diff --git a/.github/workflows/check_doc.yml b/.github/workflows/check_doc.yml index b9d552e7..dd0ef108 100644 --- a/.github/workflows/check_doc.yml +++ b/.github/workflows/check_doc.yml @@ -6,25 +6,83 @@ on: - 'docs/**' - '.readthedocs.yaml' - '.github/workflows/check_doc.yml' + - 'ppanggolin/**' + - 'setup.py' + - 'pyproject.toml' + +permissions: + contents: write jobs: - build: + generate-command-reference: + if: >- + github.event_name == 'push' && ( + contains(github.event.head_commit.modified, 'ppanggolin/') || + contains(github.event.head_commit.modified, 'setup.py') || + contains(github.event.head_commit.modified, 'pyproject.toml') || + contains(github.event.head_commit.added, 'ppanggolin/') || + contains(github.event.head_commit.added, 'setup.py') || + contains(github.event.head_commit.added, 'pyproject.toml') + ) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: install ppanggolin with python deps and doc deps + run: pip install .[doc] + + - name: regenerate command reference from CLI + run: python docs/_scripts/generate_command_reference.py + + - name: commit generated docs if changed + env: + GITHUB_REF_NAME: ${{ github.ref_name }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + if [[ -n "$(git status --porcelain -- docs/user/command_reference.md)" ]]; then + git add docs/user/command_reference.md + git commit -m "docs: regenerate command reference" + git push origin HEAD:${GITHUB_REF_NAME} + fi + + build-docs: + if: >- + github.event_name == 'push' && ( + contains(github.event.head_commit.modified, 'docs/') || + contains(github.event.head_commit.modified, '.readthedocs.yaml') || + contains(github.event.head_commit.modified, '.github/workflows/check_doc.yml') || + contains(github.event.head_commit.added, 'docs/') || + contains(github.event.head_commit.added, '.readthedocs.yaml') || + contains(github.event.head_commit.added, '.github/workflows/check_doc.yml') + ) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.12' + - name: install ppanggolin with python deps and doc deps run: pip install .[doc] - - name: Complete workflow + - name: Build documentation shell: bash -l {0} run: | cd docs/ sphinx-build -b html . build/ + # Great extra actions to compose with: # Create an artifact of the html output. - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/check_recipes.yml b/.github/workflows/check_recipes.yml index 8b9367f7..d68dab2c 100644 --- a/.github/workflows/check_recipes.yml +++ b/.github/workflows/check_recipes.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest','macos-latest'] - python-version: ['3.9','3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12', '3.13'] # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 089506f3..da0d1d62 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'macos-latest'] - python-version: ['3.9', '3.12'] + python-version: ['3.10', '3.13'] defaults: run: shell: conda run --no-capture-output -n ppanggolin bash -e {0} @@ -108,4 +108,4 @@ jobs: name: comparison-results_${{ matrix.os }}_python${{ matrix.python-version }} path: testingDataset/info_to_test/* - \ No newline at end of file + diff --git a/README.md b/README.md index 71e0aeeb..a161e492 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ By default, it uses parameters that we have found to be generally the best for w ### Input files -The file `GENOMES_FASTA_LIST` is a tsv-separated file with the following organization : +The file `GENOMES_FASTA_LIST` is a tab separated values (TSV) file with the following organization: 1. The first column contains a unique genome name **(without space)** 2. The second column contains the path to the associated FASTA file diff --git a/VERSION b/VERSION index 2bf1c1cc..f90b1afc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.1 +2.3.2 diff --git a/docs/_scripts/generate_command_reference.py b/docs/_scripts/generate_command_reference.py new file mode 100644 index 00000000..9072312e --- /dev/null +++ b/docs/_scripts/generate_command_reference.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 + +import argparse +import re +import sys +from pathlib import Path +from typing import Iterable, List, Optional, Union + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from ppanggolin.main import build_parser + + +def _format_type(action: argparse.Action) -> str: + if action.option_strings and action.nargs == 0: + return "bool" + + if isinstance(action.type, type): + return action.type.__name__ + + if action.type is not None: + type_name = getattr(action.type, "__name__", None) + if type_name in {"check_log", "restricted_float", "min_one", "filter_values"}: + return { + "check_log": "str", + "restricted_float": "float", + "min_one": "int", + "filter_values": "str", + }[type_name] + if type_name: + return type_name + if hasattr(action.type, "__call__"): + return "str" + + if action.default is not None: + if isinstance(action.default, bool): + return "bool" + return type(action.default).__name__ + + if action.const is not None: + return type(action.const).__name__ + + return "str" + + +def _format_default(action: argparse.Action) -> str: + + if action.default is None: + return "—" + if action.default is False and action.option_strings: + return "False" + if action.default is True and action.option_strings: + return "True" + + value = str(action.default) + pattern = ( + r"^(?P.*?)(?:_?DATE\d{4}-\d{2}-\d{2}_HOUR\d{2}\.\d{2}\.\d{2}_PID\d+)$" + ) + match = re.match(pattern, value) + if match: + prefix = match.group("prefix").rstrip("_") + return f"`{prefix}__`" + + if isinstance(action.default, Path): + return f"`{action.default}`" + if isinstance(action.default, (list, tuple, set)): + return f"`{str(list(action.default))}`" + if isinstance(action.default, str): + return f"`{action.default}`" + return str(action.default) + + +def _format_choices(action: argparse.Action) -> str: + if not action.choices: + return "—" + return ", ".join(f"`{choice}`" for choice in action.choices) + + +def _format_param_name(action: argparse.Action) -> str: + names = [opt for opt in action.option_strings] + if not names: + if action.metavar: + return action.metavar + return action.dest + return ", ".join(names) + + +def _describe_action(action: argparse.Action) -> dict: + return { + "name": _format_param_name(action), + "type": _format_type(action), + "default": _format_default(action), + "required": "Yes" if action.required else "No", + "choices": _format_choices(action), + "help": (action.help or "").replace("\n", " ").strip(), + "is_flag": action.option_strings and action.nargs == 0, + } + + +def _iter_action_groups( + parser: argparse.ArgumentParser, +) -> Iterable[tuple[str, list[argparse.Action]]]: + seen = set() + for action_group in parser._action_groups: + group_title = action_group.title or "Arguments" + if group_title in {"positional arguments", "options"}: + continue + + actions = [] + for action in action_group._group_actions: + if action.dest == "help": + continue + if action.dest in seen: + continue + seen.add(action.dest) + actions.append(action) + + if actions: + yield group_title, actions + + +def _generate_action_table( + title: str, actions: list[argparse.Action], command_name: Optional[str] = None +) -> list[str]: + heading = ( + title if command_name is None else f"{title} for ppanggolin {command_name}" + ) + lines = [ + f"#### {heading}", + "", + "| Parameter | Type | Default | Description |", + "|---|---|---|---|", + ] + + for action in actions: + info = _describe_action(action) + description_parts = [] + help_text = info["help"].strip() if info["help"] else "—" + description_parts.append(help_text) + + if info["required"] == "Yes": + description_parts.append("Required: Yes") + if info["choices"] != "—": + description_parts.append(f"
Choices: {info['choices']}") + + description = " ".join(part for part in description_parts if part) + lines.append( + f"| `{info['name']}` | {info['type']} | {info['default']} | {description} |" + ) + + return lines + + +def _generate_command_section( + command_name: str, parser: argparse.ArgumentParser +) -> str: + title = f"### `ppanggolin {command_name}`" + description = ( + getattr(parser, "description", None) + or "Command description is not explicitly set in this parser." + ) + lines = [ + title, + "", + description.strip(), + "", + ] + + action_groups = list(_iter_action_groups(parser)) + if not action_groups: + lines.append("No parameters are defined for this command.") + return "\n".join(lines) + "\n" + + for group_title, actions in action_groups: + lines.extend(_generate_action_table(group_title, actions, command_name)) + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + +def _collect_subcommands( + parser: argparse.ArgumentParser, +) -> List[tuple[str, argparse.ArgumentParser]]: + subparsers_action = next( + ( + group + for group in parser._action_groups + if isinstance(group, argparse._ArgumentGroup) + ), + None, + ) + subparsers = [] + for action in getattr(parser, "_actions", []): + if isinstance(action, argparse._SubParsersAction): + subparsers = action.choices.items() + break + + commands = [] + for command_name, command_parser in subparsers: + commands.append((command_name, command_parser)) + commands.sort(key=lambda item: item[0]) + return commands + + +def generate_reference(output_path: Optional[Union[Path, str]] = None) -> str: + parser = build_parser() + commands = _collect_subcommands(parser) + + sections = [ + "# Command-line reference", + "", + "Complete reference of the PPanGGOLiN command-line interface.", + "", + "This page is automatically generated from the PPanGGOLiN command-line parser.", + "", + ] + + category_order = [ + "Basic", + "Expert", + "Output", + "Regions of Genomic Plasticity", + "Analysis using reference pangenomes", + "Utility command", + ] + grouped_commands: dict[str, List[tuple[str, argparse.ArgumentParser]]] = {} + for command_name, command_parser in commands: + category = getattr(command_parser, "category", "General") + grouped_commands.setdefault(category, []).append((command_name, command_parser)) + + for category in category_order: + if category not in grouped_commands: + continue + sections.append(f"## {category}") + sections.append("") + for command_name, command_parser in grouped_commands[category]: + sections.append(_generate_command_section(command_name, command_parser)) + sections.append("") + + rendered = "\n".join(sections).strip() + "\n" + + if output_path is not None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text(rendered, encoding="utf-8") + return rendered + + +if __name__ == "__main__": + script_path = Path(__file__).resolve().parents[1] / "user" / "command_reference.md" + generate_reference(script_path) diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 00000000..702934db --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,36 @@ +.wy-table-responsive table { + width: 100% !important; + table-layout: fixed; +} + +/* .wy-table-responsive { + overflow-x: auto; +} */ + +.rst-content table.docutils td, +.rst-content table.docutils th { + white-space: normal; + /* word-wrap: break-word; */ + overflow-wrap: anywhere; + /* vertical-align: top; */ +} + +.rst-content table.docutils th:nth-child(1), +.rst-content table.docutils td:nth-child(1) { + width: 18%; +} + +.rst-content table.docutils th:nth-child(2), +.rst-content table.docutils td:nth-child(2) { + width: 12%; +} + +.rst-content table.docutils th:nth-child(3), +.rst-content table.docutils td:nth-child(3) { + width: 15%; +} + +.rst-content table.docutils th:nth-child(4), +.rst-content table.docutils td:nth-child(4) { + width: 55%; +} diff --git a/docs/conf.py b/docs/conf.py index 68a493c5..389943a4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,8 +10,23 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import subprocess +import sys from pathlib import Path + +def generate_command_reference(_app): + script = ( + Path(__file__).resolve().parent / "_scripts" / "generate_command_reference.py" + ) + if script.exists(): + subprocess.check_call([sys.executable, str(script)]) + + +def setup(app): + app.connect("builder-inited", generate_command_reference) + + # -- Project information ----------------------------------------------------- project = "PPanGGOLiN" @@ -66,3 +81,4 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] +html_css_files = ["custom.css"] diff --git a/docs/index.md b/docs/index.md index 77689b15..3797f55f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -68,6 +68,7 @@ Those RGPs can be further divided in conserved modules by panModule ([Bazin et a :maxdepth: 2 user/install +user/whatsNewInV2 user/QuickUsage/quickAnalyses user/practicalInformation user/PangenomeAnalyses/pangenomeAnalyses @@ -80,6 +81,7 @@ user/projection user/genomicContext user/MSA user/metadata +user/command_reference ``` ```{toctree} diff --git a/docs/user/Modules/modulePrediction.md b/docs/user/Modules/modulePrediction.md index ad513c2a..174bd36b 100644 --- a/docs/user/Modules/modulePrediction.md +++ b/docs/user/Modules/modulePrediction.md @@ -57,7 +57,7 @@ To execute the panModule workflow, use the following command: ```bash ppanggolin panmodule --fasta GENOME_LIST_FILE ``` -Replace `GENOME_LIST_FILE` with a tab-separated file listing the genome names, and the fasta file path of their genomic sequences as described [here](../PangenomeAnalyses/pangenomeAnnotation.md#annotate-from-fasta-files). Alternatively, you can provide a list of GFF/GBFF files as input by using the `--anno` parameter, similar to how it is used in the workflow and annotate commands. +Replace `GENOME_LIST_FILE` with a tab separated values (TSV) file listing the genome names and the FASTA file path of their genomic sequences, as described [here](../PangenomeAnalyses/pangenomeAnnotation.md#annotate-from-fasta-files). Alternatively, you can provide a list of GFF/GBFF files as input by using the `--anno` parameter, similar to how it is used in the workflow and annotate commands. The panmodule workflow predicts modules using default parameters. To fine-tune the detection, you can use the `module` command on a partitioned pangenome acquired through the `workflow` for example or use a configuration file, as described [here](../practicalInformation.md#configuration-file). @@ -72,7 +72,7 @@ The command can be used simply as such: ppanggolin module -p pangenome.h5 ``` -This will predict modules and store the results in the HDF5 pangenome file. If you wish to have descriptive tsv files, whose format is detailed [here](./moduleOutputs.md), you can use the `write_pangenome` command with the flag `--modules`: +This will predict modules and store the results in the HDF5 pangenome file. If you wish to have descriptive TSV files, whose format is detailed [here](./moduleOutputs.md), you can use the `write_pangenome` command with the flag `--modules`: ```bash ppanggolin write_pangenome -p pangenome.h5 --modules --output MYOUTPUTDIR ``` diff --git a/docs/user/PangenomeAnalyses/pangenomeAnnotation.md b/docs/user/PangenomeAnalyses/pangenomeAnnotation.md index ce46f4cb..d7d9cada 100644 --- a/docs/user/PangenomeAnalyses/pangenomeAnnotation.md +++ b/docs/user/PangenomeAnalyses/pangenomeAnnotation.md @@ -46,6 +46,7 @@ You can prevent this filtering by using the `--allow_overlap` option. Additionally, the `--kingdom archaea` option can be provided when working with archaea genomes to specify Infernal's RNA annotation model. +(annot-gff)= ### Use annotation files for your pangenome You can provide annotation files in either gff3 files or .gbk/.gbff files, or a mix of them. They should be provided through as a list in a tab-separated file that follows the same format as described for the fasta files. You can check [this example input file](https://github.com/labgem/PPanGGOLiN/blob/master/testingDataset/genomes.gbff.list). diff --git a/docs/user/RGP/rgpClustering.md b/docs/user/RGP/rgpClustering.md index c627eda4..077e99e1 100644 --- a/docs/user/RGP/rgpClustering.md +++ b/docs/user/RGP/rgpClustering.md @@ -10,17 +10,19 @@ The ppanggolin `rgp_cluster` command performs the following steps to cluster RGP There are three modes available for calculating the GRR value: `min_grr`, `max_grr`, or `incomplete_aware_grr`. - `min_grr` mode: This mode computes the number of gene families shared between two RGPs and divides it by the smaller number of gene families among the two RGPs. -- `max_grr` mode: In this mode, the number of gene families shared between two RGPs is calculated and divided by the larger number of gene families among the two RGPs. +- `max_grr` mode: In this mode, the number of gene families shared between two RGPs is calculated and divided by the larger number of gene families between the two RGPs. - `incomplete_aware_grr` (default) mode: If at least one RGP is considered incomplete, which typically happens when it is located at the border of a contig, the `min_grr` mode is used. Otherwise, the `max_grr` mode is applied. This mode is useful to correctly cluster incomplete RGP. -The resulting RGP clusters are stored in a tsv file with the following columns: +By default, the output files are written in a directory named `rgp_clustering` created in the working directory, which can be changed with the `--output` parameter. The files themselves are named after the `--basename` parameter (`rgp_cluster` by default). -| column | description | -|---------|------------------------------| -|RGP|The unique region identifier.| -|cluster|The cluster id of the RGP.| -|spot_id|The spot ID of the RGP.| +The resulting RGP clusters are stored in a TSV file (`rgp_cluster.tsv`) with the following columns: +| column | description | +|---------|-------------------------------| +| RGP | The unique region identifier. | +| cluster | The cluster id of the RGP. | +| spot_id | The spot ID of the RGP. | -The command also generates the RGP graph in the gexf format, which can be used to explore the RGP clusters along with their spots of insertion. In this graph, identical RGPs with the same family content and with the same spot are merged into a single node to simplify the graph representation. This feature can be disable with the parameter `--no_identical_rgp_merging`. + +The command also generates an RGP graph in GEXF or GraphML format, which can be used to explore RGP clusters together with their associated spots of insertion. In this graph, identical RGPs with the same family content are merged into a single node to simplify the graph representation. This feature can be disabled with the parameter `--no_identical_rgp_merging`. diff --git a/docs/user/RGP/rgpOutputs.md b/docs/user/RGP/rgpOutputs.md index abfe41e9..ceb04763 100644 --- a/docs/user/RGP/rgpOutputs.md +++ b/docs/user/RGP/rgpOutputs.md @@ -2,8 +2,8 @@ ### RGP -The `regions_of_genomic_plasticity.tsv` is a tsv file that lists all the detected Regions of Genome Plasticity. This -requires to have run the RGP detection analysis by either using the `panrgp` command or the `rgp` command. +The `regions_of_genomic_plasticity.tsv` is a TSV file that lists all the detected Regions of Genome Plasticity. This +requires having run the RGP detection analysis by either using the `panrgp` command or the `rgp` command. It can be written with the following command: @@ -48,7 +48,7 @@ ppanggolin write_pangenome -p pangenome.h5 --regions_families -o rgp_outputs ### Spots -The `spots.tsv` is a tsv file that links the spots in `summarize_spots.tsv` with the RGPs +The `spots.tsv` is a TSV file that links the spots in `summarize_spots.tsv` with the RGPs in `regions_of_genomic_plasticity.tsv`. It can be created with the following command: @@ -64,7 +64,7 @@ ppanggolin write_pangenome -p pangenome.h5 --spots -o rgp_outputs ### Summarize spots -The `summarize_spots.tsv` file is a tsv file that will associate each spot with multiple metrics that can indicate the +The `summarize_spots.tsv` file is a TSV file that will associate each spot with multiple metrics that can indicate the dynamic of the spot. It can be created with the following command: diff --git a/docs/user/align.md b/docs/user/align.md index decdf4ce..31aaa4fe 100644 --- a/docs/user/align.md +++ b/docs/user/align.md @@ -9,6 +9,10 @@ This command is used as follows: ppanggolin align -p pangenome.h5 -o MYOUTPUTDIR --sequences MY_SEQUENCES_OF_INTEREST.fasta ``` + +An example file containing proteins of interest from *Chlamydia trachomatis* can be found in the [testingDataset](https://github.com/labgem/PPanGGOLiN/blob/master/testingDataset/some_chlam_proteins.fasta) directory. + + ## Output files By default the command creates two output files: diff --git a/docs/user/command_reference.md b/docs/user/command_reference.md new file mode 100644 index 00000000..11c58008 --- /dev/null +++ b/docs/user/command_reference.md @@ -0,0 +1,1019 @@ +# Command-line reference + +Complete reference of the PPanGGOLiN command-line interface. + +This page is automatically generated from the PPanGGOLiN command-line parser. + +## Basic + +### `ppanggolin all` + +Easy workflow to run all possible analysis + +#### Input arguments for ppanggolin all + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--fasta` | Path | — | A tab-separated file listing genome names and the FASTA file paths of their genomic sequence(s) (the FASTA files can be compressed). One line per genome. This option can be used alone. | +| `--anno` | Path | — | A tab-separated file listing genome names and the GFF file paths of their annotations (the GFF files can be compressed). One line per genome. This option can be used alone if the FASTA sequences are already present in the GFF files; otherwise, --fasta must be used. | +| `--clusters` | Path | — | A tab-separated file listing cluster names, gene IDs, and optionally whether they are fragments. | + +#### Optional arguments for ppanggolin all + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-o, --output` | Path | `ppanggolin_output__` | Output directory. | +| `--basename` | str | `pangenome` | Basename for the output file. | +| `--rarefaction` | bool | False | Compute the rarefaction curves. Warning: this can be time consuming. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified and using annotation files (--anno), the translation table information found in the annotation files will be used. Otherwise, the default genetic code 11 will be used. | +| `--kingdom` | lower | `bacteria` | Kingdom to which the prokaryote belongs, to determine which models to use for rRNA annotation.
Choices: `bacteria`, `archaea` | +| `--mode` | str | `1` | MMseqs2 clustering mode: 0 = Setcover, 1 = single linkage (or connected component), 2 = CD-HIT-like, 3 = CD-HIT-like (lowmem).
Choices: `0`, `1`, `2`, `3` | +| `--coverage` | float | 0.8 | Minimum coverage of the alignment for two proteins to be in the same cluster. | +| `--identity` | float | 0.8 | Minimum identity percentage for two proteins to be in the same cluster. | +| `--infer_singletons` | bool | False | Use this option together with --clusters. If a gene is not present in the provided clustering result file, it will be assigned to its own unique cluster as a singleton. | +| `--use_pseudo` | bool | False | In the context of provided annotation, use this option to read pseudogenes. (Default behavior is to ignore them). | +| `-K, --nb_of_partitions` | int | -1 | Number of partitions to use. Must be at least 2. If under 2, it will be detected automatically. | +| `--no_defrag` | bool | False | Do not realign gene families to link fragments with their non-fragmented gene family. | +| `--no_flat_files` | bool | False | Generate only the HDF5 pangenome file. | +| `--tmpdir` | str | `/tmp` | Directory for storing temporary files. | + +#### Common arguments for ppanggolin all + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin panmodule` + +Easy workflow to run a pangenome analysis with module prediction + +#### Input arguments for ppanggolin panmodule + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--fasta` | Path | — | A tab-separated file listing genome names and the FASTA file paths of their genomic sequence(s) (the FASTA files can be compressed). One line per genome. This option can be used alone. | +| `--anno` | Path | — | A tab-separated file listing genome names and the GFF file paths of their annotations (the GFF files can be compressed). One line per genome. This option can be used alone if the FASTA sequences are already present in the GFF files; otherwise, --fasta must be used. | +| `--clusters` | Path | — | A tab-separated file listing cluster names, gene IDs, and optionally whether they are fragments. | + +#### Optional arguments for ppanggolin panmodule + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-o, --output` | Path | `ppanggolin_output__` | Output directory. | +| `--basename` | str | `pangenome` | Basename for the output file. | +| `--rarefaction` | bool | False | Compute the rarefaction curves. Warning: this can be time consuming. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified and using annotation files (--anno), the translation table information found in the annotation files will be used. Otherwise, the default genetic code 11 will be used. | +| `--kingdom` | lower | `bacteria` | Kingdom to which the prokaryote belongs, to determine which models to use for rRNA annotation.
Choices: `bacteria`, `archaea` | +| `--mode` | str | `1` | MMseqs2 clustering mode: 0 = Setcover, 1 = single linkage (or connected component), 2 = CD-HIT-like, 3 = CD-HIT-like (lowmem).
Choices: `0`, `1`, `2`, `3` | +| `--coverage` | float | 0.8 | Minimum coverage of the alignment for two proteins to be in the same cluster. | +| `--identity` | float | 0.8 | Minimum identity percentage for two proteins to be in the same cluster. | +| `--infer_singletons` | bool | False | Use this option together with --clusters. If a gene is not present in the provided clustering result file, it will be assigned to its own unique cluster as a singleton. | +| `--use_pseudo` | bool | False | In the context of provided annotation, use this option to read pseudogenes. (Default behavior is to ignore them). | +| `-K, --nb_of_partitions` | int | -1 | Number of partitions to use. Must be at least 2. If under 2, it will be detected automatically. | +| `--no_defrag` | bool | False | Do not realign gene families to link fragments with their non-fragmented gene family. | +| `--no_flat_files` | bool | False | Generate only the HDF5 pangenome file. | +| `--tmpdir` | str | `/tmp` | Directory for storing temporary files. | + +#### Common arguments for ppanggolin panmodule + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin panrgp` + +Easy workflow to run a pangenome analysis with genomic islands and spots of insertion detection + +#### Input arguments for ppanggolin panrgp + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--fasta` | Path | — | A tab-separated file listing genome names and the FASTA file paths of their genomic sequence(s) (the FASTA files can be compressed). One line per genome. This option can be used alone. | +| `--anno` | Path | — | A tab-separated file listing genome names and the GFF file paths of their annotations (the GFF files can be compressed). One line per genome. This option can be used alone if the FASTA sequences are already present in the GFF files; otherwise, --fasta must be used. | +| `--clusters` | Path | — | A tab-separated file listing cluster names, gene IDs, and optionally whether they are fragments. | + +#### Optional arguments for ppanggolin panrgp + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-o, --output` | Path | `ppanggolin_output__` | Output directory. | +| `--basename` | str | `pangenome` | Basename for the output file. | +| `--rarefaction` | bool | False | Compute the rarefaction curves. Warning: this can be time consuming. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified and using annotation files (--anno), the translation table information found in the annotation files will be used. Otherwise, the default genetic code 11 will be used. | +| `--kingdom` | lower | `bacteria` | Kingdom to which the prokaryote belongs, to determine which models to use for rRNA annotation.
Choices: `bacteria`, `archaea` | +| `--mode` | str | `1` | MMseqs2 clustering mode: 0 = Setcover, 1 = single linkage (or connected component), 2 = CD-HIT-like, 3 = CD-HIT-like (lowmem).
Choices: `0`, `1`, `2`, `3` | +| `--coverage` | float | 0.8 | Minimum coverage of the alignment for two proteins to be in the same cluster. | +| `--identity` | float | 0.8 | Minimum identity percentage for two proteins to be in the same cluster. | +| `--infer_singletons` | bool | False | Use this option together with --clusters. If a gene is not present in the provided clustering result file, it will be assigned to its own unique cluster as a singleton. | +| `--use_pseudo` | bool | False | In the context of provided annotation, use this option to read pseudogenes. (Default behavior is to ignore them). | +| `-K, --nb_of_partitions` | int | -1 | Number of partitions to use. Must be at least 2. If under 2, it will be detected automatically. | +| `--no_defrag` | bool | False | Do not realign gene families to link fragments with their non-fragmented gene family. | +| `--no_flat_files` | bool | False | Generate only the HDF5 pangenome file. | +| `--tmpdir` | str | `/tmp` | Directory for storing temporary files. | + +#### Common arguments for ppanggolin panrgp + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin workflow` + +Easy workflow to run a pangenome analysis in one go + +#### Input arguments for ppanggolin workflow + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--fasta` | Path | — | A tab-separated file listing genome names and the FASTA file paths of their genomic sequence(s) (the FASTA files can be compressed). One line per genome. This option can be used alone. | +| `--anno` | Path | — | A tab-separated file listing genome names and the GFF file paths of their annotations (the GFF files can be compressed). One line per genome. This option can be used alone if the FASTA sequences are already present in the GFF files; otherwise, --fasta must be used. | +| `--clusters` | Path | — | A tab-separated file listing cluster names, gene IDs, and optionally whether they are fragments. | + +#### Optional arguments for ppanggolin workflow + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-o, --output` | Path | `ppanggolin_output__` | Output directory. | +| `--basename` | str | `pangenome` | Basename for the output file. | +| `--rarefaction` | bool | False | Compute the rarefaction curves. Warning: this can be time consuming. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified and using annotation files (--anno), the translation table information found in the annotation files will be used. Otherwise, the default genetic code 11 will be used. | +| `--kingdom` | lower | `bacteria` | Kingdom to which the prokaryote belongs, to determine which models to use for rRNA annotation.
Choices: `bacteria`, `archaea` | +| `--mode` | str | `1` | MMseqs2 clustering mode: 0 = Setcover, 1 = single linkage (or connected component), 2 = CD-HIT-like, 3 = CD-HIT-like (lowmem).
Choices: `0`, `1`, `2`, `3` | +| `--coverage` | float | 0.8 | Minimum coverage of the alignment for two proteins to be in the same cluster. | +| `--identity` | float | 0.8 | Minimum identity percentage for two proteins to be in the same cluster. | +| `--infer_singletons` | bool | False | Use this option together with --clusters. If a gene is not present in the provided clustering result file, it will be assigned to its own unique cluster as a singleton. | +| `--use_pseudo` | bool | False | In the context of provided annotation, use this option to read pseudogenes. (Default behavior is to ignore them). | +| `-K, --nb_of_partitions` | int | -1 | Number of partitions to use. Must be at least 2. If under 2, it will be detected automatically. | +| `--no_defrag` | bool | False | Do not realign gene families to link fragments with their non-fragmented gene family. | +| `--no_flat_files` | bool | False | Generate only the HDF5 pangenome file. | +| `--tmpdir` | str | `/tmp` | Directory for storing temporary files. | + +#### Common arguments for ppanggolin workflow + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +## Expert + +### `ppanggolin annotate` + +Annotate genomes + +#### Required arguments for ppanggolin annotate + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--fasta` | Path | — | A tab-separated file listing genome names and the FASTA file paths of their genomic sequences (the FASTA files can be compressed with gzip). One line per genome. | +| `--anno` | Path | — | A tab-separated file listing genome names and the GFF/GBFF file paths of their annotations (the files can be compressed with gzip). One line per genome. If this is provided, those annotations will be used. | + +#### Optional arguments for ppanggolin annotate + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-o, --output` | Path | `ppanggolin_output__` | Output directory. | +| `--allow_overlap` | bool | False | Use this option to keep genes overlapping with RNA features. | +| `--norna` | bool | False | Use this option to avoid annotating RNA features. | +| `--kingdom` | lower | `bacteria` | Kingdom to which the prokaryote belongs, to determine which models to use for rRNA annotation.
Choices: `bacteria`, `archaea` | +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified and using annotation files (--anno), the translation table information found in the annotation files will be used. Otherwise, the default genetic code 11 will be used. | +| `--basename` | str | `pangenome` | Basename for the output file. | +| `--use_pseudo` | bool | False | In the context of provided annotation, use this option to read pseudogenes. (Default behavior is to ignore them). | +| `-p, --prodigal_procedure` | lower | — | Allow forcing the Prodigal procedure. If not specified, PPanGGOLiN will decide based on contig length.
Choices: `single`, `meta` | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--tmpdir` | str | `/tmp` | Directory for storing temporary files. | + +#### Common arguments for ppanggolin annotate + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin cluster` + +Cluster genes into gene families + +#### Required arguments for ppanggolin cluster + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | + +#### Clustering arguments for ppanggolin cluster + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--identity` | float | 0.8 | Minimum identity percentage for two proteins to be in the same cluster. | +| `--coverage` | float | 0.8 | Minimum coverage of the alignment for two proteins to be in the same cluster. | +| `--mode` | str | `1` | MMseqs2 clustering mode: 0 = Setcover, 1 = single linkage (or connected component), 2 = CD-HIT-like, 3 = CD-HIT-like (lowmem).
Choices: `0`, `1`, `2`, `3` | +| `--no_defrag` | bool | False | Do not use the defragmentation strategy to link potential fragments with their original gene family. | + +#### Read clustering arguments for ppanggolin cluster + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--clusters` | Path | — | A tab-separated list containing the result of a clustering. One line per gene. The first column is the cluster ID and the second is the gene ID. | +| `--infer_singletons` | bool | False | When reading a clustering result with --clusters, if a gene is not in the provided file, it will be placed in a cluster where the gene is the only member. | + +#### Optional arguments for ppanggolin cluster + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified, the translation table used when building the pangenome will be used. This can be accessed using 'ppanggolin info'. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--tmpdir` | Path | `/tmp` | Directory for storing temporary files. | +| `--keep_tmp` | bool | False | Keep temporary files (useful for debugging). | + +#### Common arguments for ppanggolin cluster + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin graph` + +Create the pangenome graph + +#### Required arguments for ppanggolin graph + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | + +#### Optional arguments for ppanggolin graph + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-r, --remove_high_copy_number` | int | 0 | Positive Number: Remove families having a number of copy of gene in a single genome above or equal to this threshold in at least one genome (0 or negative values are ignored). | + +#### Common arguments for ppanggolin graph + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin metadata` + +Add metadata to elements in yout pangenome + +#### Required arguments for ppanggolin metadata + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | +| `-m, --metadata` | Path | — | Metadata in TSV file. See our github for more detail about format | +| `-s, --source` | str | — | Name of the metadata source | +| `-a, --assign` | str | — | Select to which pangenome element metadata will be assigned
Choices: `families`, `genomes`, `contigs`, `genes`, `RGPs`, `spots`, `modules` | + +#### Optional arguments for ppanggolin metadata + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--omit` | bool | False | Allow to pass if a key in metadata is not find in pangenome | + +#### Common arguments for ppanggolin metadata + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin partition` + +Partition the pangenome graph + +#### Required arguments for ppanggolin partition + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | + +#### Optional arguments for ppanggolin partition + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-b, --beta` | float | 2.5 | beta is the strength of the smoothing using the graph topology during partitioning. 0 will deactivate spatial smoothing. | +| `-ms, --max_degree_smoothing` | float | 10 | max. degree of the nodes to be included in the smoothing process. | +| `-o, --output` | Path | `ppanggolin_output__` | Output directory. | +| `-fd, --free_dispersion` | bool | False | use if the dispersion around the centroid vector of each partition during must be free. It will be the same for all genomes by default. | +| `-ck, --chunk_size` | int | 500 | Size of the chunks when performing partitioning using chunks of genomes. Chunk partitioning will be used automatically if the number of genomes is above this number. | +| `-K, --nb_of_partitions` | int | -1 | Number of partitions to use. Must be at least 2. If under 2, it will be detected automatically. | +| `-Kmm, --krange` | int | `[3, 20]` | Range of K values to test when detecting K automatically. | +| `-im, --ICL_margin` | float | 0.05 | K is detected automatically by maximizing ICL. However at some point the ICL reaches a plateau. Therefore we are looking for the minimal value of K without significant gain from the larger values of K measured by ICL. For that we take the lowest K that is found within a given 'margin' of the maximal ICL value. Basically, change this option only if you truly understand it, otherwise just leave it be. | +| `--draw_ICL` | bool | False | Use if you want to draw the ICL curve for all the tested K values. Will not be done if K is given. | +| `--keep_tmp_files` | bool | False | Use if you want to keep the temporary NEM files | +| `-se, --seed` | int | 42 | Seed used to generate random numbers. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--tmpdir` | str | `/tmp` | Directory for storing temporary files. | + +#### Common arguments for ppanggolin partition + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin rarefaction` + +Compute the rarefaction curve of the pangenome + +#### Required arguments for ppanggolin rarefaction + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | + +#### Optional arguments for ppanggolin rarefaction + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-b, --beta` | float | 2.5 | beta is the strength of the smoothing using the graph topology during partitioning. 0 will deactivate spatial smoothing. | +| `--depth` | int | 30 | Number of samplings at each sampling point | +| `--min` | int | 1 | Minimum number of genomes in a sample | +| `--max` | float | 100 | Maximum number of genomes in a sample (if above the number of provided genomes, the provided genomes will be the maximum) | +| `-ms, --max_degree_smoothing` | float | 10 | max. degree of the nodes to be included in the smoothing process. | +| `-o, --output` | Path | `ppanggolin_output__` | Output directory. | +| `-fd, --free_dispersion` | bool | False | use if the dispersion around the centroid vector of each partition during must be free. It will be the same for all genomes by default. | +| `-ck, --chunk_size` | int | 500 | Size of the chunks when performing partitioning using chunks of genomes. Chunk partitioning will be used automatically if the number of genomes is above this number. | +| `-K, --nb_of_partitions` | int | -1 | Number of partitions to use. Must be at least 2. By default reuse K if it exists else compute it. | +| `--reestimate_K` | bool | False | Will recompute the number of partitions for each sample (between the values provided by --krange) (VERY intensive. Can take a long time.) | +| `-Kmm, --krange` | int | `[3, -1]` | Range of K values to test when detecting K automatically. Default between 3 and the K previously computed if there is one, or 20 if there are none. | +| `--soft_core` | float | 0.95 | Soft core threshold | +| `-se, --seed` | int | 42 | seed used to generate random numbers | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--tmpdir` | str | `/tmp` | Directory for storing temporary files. | + +#### Common arguments for ppanggolin rarefaction + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +## Output + +### `ppanggolin draw` + +Draw figures representing the pangenome through different aspects + +#### Required arguments for ppanggolin draw + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | + +#### Optional arguments for ppanggolin draw + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-o, --output` | Path | `ppanggolin_output__` | Output directory. | +| `--tile_plot` | bool | False | draw the tile plot of the pangenome | +| `--soft_core` | float | 0.95 | Soft core threshold to use | +| `--ucurve` | bool | False | draw the U-curve of the pangenome | +| `--draw_spots` | bool | False | draw plots for spots of the pangenome | +| `--spots` | str | `all` | a comma-separated list of spots to draw (or 'all' to draw all spots, or 'synteny' to draw spots with different RGP syntenies). | +| `--nocloud` | bool | False | Do not draw the cloud genes in the tile plot | +| `--add_dendrogram` | bool | False | Include a dendrogram for genomes in the tile plot based on the presence/absence of gene families. | +| `--add_metadata` | bool | False | Display gene metadata as hover text for each cell in the tile plot. | +| `--metadata_sources` | str | — | Which source of metadata should be written in the tile plot. By default all metadata sources are included. | + +#### Common arguments for ppanggolin draw + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin fasta` + +Writes fasta files for different elements of the pangenome. + +#### Required arguments for ppanggolin fasta + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | +| `-o, --output` | Path | — | Output directory where the file(s) will be written. Required: Yes | + +#### Contextually required arguments for ppanggolin fasta + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--fasta` | Path | — | A tab-separated file listing genome names and the FASTA file paths of their genomic sequence(s) (the fastas can be compressed with gzip). One line per genome. | +| `--anno` | Path | — | A tab-separated file listing genome names and the GFF/GBFF file paths of their annotations annotations (the files can be compressed with gzip). One line per genome. If this is provided, those annotations will be used. | + +#### Output file for ppanggolin fasta + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--genes` | str | — | Write all nucleotide CDS sequences. Possible values are all, persistent, shell, cloud, rgp, softcore, core, module_X with X being a module id. | +| `--proteins` | str | — | Write representative amino acid sequences of genes. Possible values are all, persistent, shell, cloud, rgp, softcore, core, module_X with X being a module id. | +| `--prot_families` | str | — | Write representative amino acid sequences of gene families. Possible values are all, persistent, shell, cloud, rgp, softcore, core, module_X with X being a module id. | +| `--gene_families` | str | — | Write representative nucleotide sequences of gene families. Possible values are all, persistent, shell, cloud, rgp, softcore, core, module_X with X being a module id. | + +#### Optional arguments for ppanggolin fasta + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--regions` | str | — | Write the RGP nucleotide sequences (requires --anno or --fasta used to compute the pangenome to be given)
Choices: `all`, `complete` | +| `--soft_core` | float | 0.95 | Soft core threshold to use if 'softcore' partition is chosen | +| `--compress` | bool | False | Compress the files in .gz | +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified, the translation table used when building the pangenome will be used. This can be accessed using 'ppanggolin info'. | +| `--cpu` | int | 1 | Number of available threads. | +| `--tmpdir` | Path | `/tmp` | Directory for storing temporary files. | +| `--keep_tmp` | bool | False | Keeping temporary files (useful for debugging). | + +#### Common arguments for ppanggolin fasta + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin info` + +Prints information about a given pangenome graph file. + +#### Required arguments for ppanggolin info + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. Required: Yes | + +#### Information Display Options (default: all) for ppanggolin info + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-a, --parameters` | bool | False | Display the parameters used or computed for each step of pangenome generation | +| `-c, --content` | bool | False | Display detailed information about the pangenome's content | +| `-s, --status` | bool | False | Display information about the statuses of different elements in the pangenome, indicating what has been computed or not | +| `-m, --metadata` | bool | False | Display a summary of the metadata saved in the pangenome | + +#### Common arguments for ppanggolin info + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin metrics` + +Compute several metrics on a given pangenome. + +#### Required arguments for ppanggolin metrics + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | + +#### Input file for ppanggolin metrics + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--genome_fluidity` | bool | False | Compute the pangenome genomic fluidity. | + +#### Optional arguments for ppanggolin metrics + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--no_print_info` | bool | False | Suppress printing the metrics result. Metrics are saved in the pangenome and viewable using 'ppanggolin info'. | +| `--recompute_metrics` | bool | False | Force re-computation of metrics if already computed. | + +#### Common arguments for ppanggolin metrics + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin write_genomes` + +Writes 'flat' files that represent the genomes along with their associated pangenome elements. + +#### Required arguments for ppanggolin write_genomes + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | +| `-o, --output` | Path | — | Output directory where the file(s) will be written. Required: Yes | + +#### Optional arguments for ppanggolin write_genomes + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--table` | bool | False | Generate a tsv file for each genome with pangenome annotations. | +| `--gff` | bool | False | Generate a gff file for each genome containing pangenome annotations. | +| `--proksee` | bool | False | Generate JSON map files for PROKSEE for each genome containing pangenome annotations to be used in proksee. | +| `--compress` | bool | False | Compress the files in .gz | +| `--genomes` | str | `all` | Specify the genomes for which to generate output. You can provide a list of genome names either directly in the command line separated by commas, or by referencing a file containing the list of genome names, with one name per line. | +| `--add_metadata` | bool | False | Include metadata information in the output files if any have been added to pangenome elements (see ppanggolin metadata command). | +| `--metadata_sources` | str | — | Which source of metadata should be written. By default all metadata sources are included. | +| `--metadata_sep` | str | `|` | The separator used to join multiple metadata values for elements with multiple metadata values from the same source. This character should not appear in metadata values. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | + +#### Contextually required arguments for ppanggolin write_genomes + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--fasta` | Path | — | A tab-separated file listing genome names and the FASTA file paths of their genomic sequence(s) (the fastas can be compressed with gzip). One line per genome. | +| `--anno` | Path | — | A tab-separated file listing genome names and the GFF/GBFF file paths of their annotations annotations (the files can be compressed with gzip). One line per genome. If this is provided, those annotations will be used. | + +#### Common arguments for ppanggolin write_genomes + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin write_metadata` + +Writes 'TSV' files that represent the metadata associated with elements of the pangenome. + +#### Required arguments for ppanggolin write_metadata + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | +| `-o, --output` | Path | — | Output directory where the file(s) will be written. Required: Yes | + +#### Optional arguments for ppanggolin write_metadata + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--compress` | bool | False | Compress the files in .gz | +| `-e, --pangenome_elements` | set | `['families', 'RGPs', 'modules', 'genomes', 'genes', 'contigs', 'spots']` | Specify pangenome elements for which to write metadata. default is all element with metadata.
Choices: `families`, `RGPs`, `modules`, `genomes`, `genes`, `contigs`, `spots` | +| `-s, --metadata_sources` | str | — | Which source of metadata should be written. By default all metadata sources are included. | + +#### Common arguments for ppanggolin write_metadata + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin write_pangenome` + +Writes 'flat' files that represent the pangenome and its elements for use with other software. + +#### Required arguments for ppanggolin write_pangenome + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | +| `-o, --output` | Path | — | Output directory where the file(s) will be written. Required: Yes | + +#### Optional arguments for ppanggolin write_pangenome + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--soft_core` | float | 0.95 | Soft core threshold to use | +| `--dup_margin` | float | 0.05 | Minimum ratio of genomes in which the family must have multiple genes for it to be considered 'duplicated' | +| `--gexf` | bool | False | Generate a detailed GEXF file with all genes and annotations for each family | +| `--light_gexf` | bool | False | Generate a simplified GEXF file with basic gene family information | +| `--gt` | bool | False | Generate a simplified graph-tool GT file with basic gene family and pangenome organism strains information | +| `--json` | bool | False | Export the graph in JSON file format | +| `--csv` | bool | False | Export gene presence/absence in CSV format (compatible with Roary). Uses partitions as alternative gene IDs if available. | +| `--Rtab` | bool | False | Export gene presence/absence as a tabular binary matrix | +| `--stats` | bool | False | Generate genome statistics ('genome_statistics.tsv') and duplication summary of persistent families ('mean_persistent_duplication.tsv') | +| `--partitions` | bool | False | Generate one file per partition listing the gene families it contains | +| `--families_tsv` | bool | False | Write a TSV file mapping genes to their corresponding gene families | +| `--regions` | bool | False | Write predicted RGPs (Regions of Genomic Plasticity) and metrics to 'plastic_regions.tsv' | +| `--regions_families` | bool | False | Write a TSV file mapping each RGP to its gene family content in 'rgp_families.tsv' | +| `--spots` | bool | False | Write spot summary and list all RGPs (Regions of Genomic Plasticity) per spot | +| `--borders` | bool | False | List all borders of each spot | +| `--modules` | bool | False | Write a TSV file listing functional modules and their associated gene families | +| `--spot_modules` | bool | False | Generate two files comparing module presence across spots | +| `--compress` | bool | False | Compress output files using gzip (.gz) | +| `-c, --cpu` | int | 1 | Number of available CPUs. | + +#### Common arguments for ppanggolin write_pangenome + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +## Regions of Genomic Plasticity + +### `ppanggolin module` + +Predicts functional modules in your pangenome. + +#### Required arguments for ppanggolin module + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | + +#### Optional arguments for ppanggolin module + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--size` | int | 3 | Minimal number of gene family in a module | +| `--dup_margin` | float | 0.05 | minimum ratio of genomes in which the family must have multiple genes for it to be considered 'duplicated' | +| `-m, --min_presence` | int | 2 | Minimum number of times the module needs to be present in the pangenome to be reported. Increasing it will improve precision but lower sensitivity. | +| `-t, --transitive` | int | 4 | Size of the transitive closure used to build the graph. This indicates the number of non related genes allowed in-between two related genes. Increasing it will improve precision but lower sensitivity a little. | +| `-s, --jaccard` | float | 0.85 | Minimum Jaccard similarity used to filter edges between gene families. Increasing it will improve precision but lower sensitivity a lot. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | + +#### Common arguments for ppanggolin module + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin rgp` + +Predicts Regions of Genomic Plasticity in the genomes of your pangenome. + +#### Required arguments for ppanggolin rgp + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | + +#### Optional arguments for ppanggolin rgp + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--persistent_penalty` | int | 3 | Penalty score to apply to persistent genes | +| `--variable_gain` | int | 1 | Gain score to apply to variable genes | +| `--min_score` | int | 4 | Minimal score wanted for considering a region as being a RGP | +| `--min_length` | int | 3000 | Minimum length (bp) of a region to be considered a RGP | +| `--dup_margin` | float | 0.05 | Minimum ratio of genomes where the family is present in which the family must have multiple genes for it to be considered 'duplicated' | + +#### Common arguments for ppanggolin rgp + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin rgp_cluster` + +Cluster RGPs based on their gene families. + +#### Required arguments for ppanggolin rgp_cluster + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. Required: Yes | + +#### Optional arguments for ppanggolin rgp_cluster + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--grr_cutoff` | float | 0.8 | Min gene repertoire relatedness metric used in the rgp clustering | +| `--grr_metric` | str | `incomplete_aware_grr` | The grr (Gene Repertoire Relatedness) is used to assess the similarity between two RGPs based on their gene families.There are three different modes for calculating the grr value: 'min_grr', 'max_grr' or 'incomplete_aware_grr'.'min_grr': Computes the number of gene families shared between the two RGPs and divides it by the smaller number of gene families among the two RGPs.'max_grr': Calculates the number of gene families shared between the two RGPs and divides it by the larger number of gene families among the two RGPs.'incomplete_aware_grr' (default): If at least one RGP is considered incomplete, which occurs when it is located at the border of a contig,the 'min_grr' mode is used. Otherwise, the 'max_grr' mode is applied.
Choices: `incomplete_aware_grr`, `min_grr`, `max_grr` | +| `--ignore_incomplete_rgp` | bool | False | Do not cluster RGPs located on a contig border which are likely incomplete. | +| `--no_identical_rgp_merging` | bool | False | Do not merge in one node identical RGP (i.e. having the same family content) before clustering. | +| `--basename` | str | `rgp_cluster` | Basename for the output file. | +| `-o, --output` | Path | `rgp_clustering` | Output directory. | +| `--graph_formats` | str | `['gexf', 'graphml']` | Format of the output graph.
Choices: `gexf`, `graphml` | +| `--add_metadata` | bool | False | Include metadata information in the output files if any have been added to pangenome elements (see ppanggolin metadata command). | +| `--metadata_sources` | str | — | Which source of metadata should be written. By default all metadata sources are included. | +| `--metadata_sep` | str | `|` | The separator used to join multiple metadata values for elements with multiple metadata values from the same source. This character should not appear in metadata values. | + +#### Common arguments for ppanggolin rgp_cluster + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin spot` + +Predicts spots in your pangenome. + +#### Required arguments for ppanggolin spot + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | + +#### Optional arguments for ppanggolin spot + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-o, --output` | Path | `ppanggolin_output__` | Output directory. | +| `--spot_graph` | bool | False | Writes a graph of pairs of blocks of single copy markers flanking RGPs, supposedly belonging to the same hotspot | +| `--overlapping_match` | int | 2 | The number of 'missing' persistent genes allowed when comparing flanking genes during hotspot computations | +| `--set_size` | int | 3 | Number of single copy markers to use as flanking genes for a RGP during hotspot computation | +| `--exact_match_size` | int | 1 | Number of perfectly matching flanking single copy markers required to associate RGPs during hotspot computation (Ex: If set to 1, two RGPs are in the same hotspot if both their 1st flanking genes are the same) | +| `--graph_formats` | str | `['gexf']` | Format of the output graph.
Choices: `gexf`, `graphml` | + +#### Common arguments for ppanggolin spot + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +## Analysis using reference pangenomes + +### `ppanggolin align` + +Aligns a genome or a set of proteins to the pangenome gene families and predicts information from it. + +#### Required arguments for ppanggolin align + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-S, --sequences` | Path | — | Sequences (nucleotides or amino acids) to align against the pangenome gene families. | +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | +| `-o, --output` | Path | — | Output directory where the file(s) will be written. Required: Yes | + +#### Optional arguments for ppanggolin align + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--no_defrag` | bool | False | Do not realign gene families to link fragments with their non-fragmented gene family. (Default: False) | +| `--identity` | float | 0.5 | Minimum identity percentage threshold. | +| `--coverage` | float | 0.8 | Minimum coverage percentage threshold. | +| `--fast` | bool | False | Use representative sequences of gene families for input gene alignment. This option is faster but may be less sensitive. By default, all pangenome genes are used. | +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified, the translation table used when building the pangenome will be used. This can be accessed using 'ppanggolin info'. | +| `--getinfo` | bool | False | Use this option to extract information related to the best hit of each query, such as the RGP it is in or the spots it belongs to. | +| `--draw_related` | bool | False | Draw figures and provide graphs in GEXF format for the spots associated with the input sequences. | +| `--use_pseudo` | bool | False | In the context of provided annotation, use this option to read pseudogenes. (Default behavior is to ignore them). | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--tmpdir` | Path | `/tmp` | Directory for storing temporary files. | +| `--keep_tmp` | bool | False | Keep temporary files (useful for debugging). | + +#### Common arguments for ppanggolin align + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin context` + +Local genomic context analysis. + +#### Required arguments for ppanggolin context + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | +| `-o, --output` | Path | `ppanggolin_context__` | Output directory where the file(s) will be written. | + +#### Input file for ppanggolin context + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-S, --sequences` | Path | — | FASTA file with the sequences of interest. | +| `-F, --family` | Path | — | List of family IDs of interest from the pangenome. | + +#### Optional arguments for ppanggolin context + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-t, --transitive` | int | 4 | Size of the transitive closure used to build the graph. This indicates the number of unrelated genes allowed between two related genes. Increasing it will improve precision but lower sensitivity slightly. | +| `-w, --window_size` | int | 5 | Number of neighboring genes considered on each side of a gene of interest when searching for conserved genomic contexts. | +| `-s, --jaccard` | float | 0.85 | Minimum Jaccard similarity used to filter edges between gene families. Increasing it will improve precision but lower sensitivity considerably. | +| `--graph_format` | str | `graphml` | Format of the context graph. Can be GEXF or GraphML.
Choices: `gexf`, `graphml` | + +#### Alignment arguments for ppanggolin context + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--no_defrag` | bool | False | Do not realign gene families to link fragments with their non-fragmented gene family. | +| `--fast` | bool | False | Use representative sequences of gene families for input gene alignment. This option is recommended for faster processing but may be less sensitive. By default, all pangenome genes are used for alignment. | +| `--identity` | float | 0.8 | Minimum identity percentage threshold. | +| `--coverage` | float | 0.8 | Minimum coverage percentage threshold. | +| `--translation_table` | int | 11 | The translation table to use when the input sequences are nucleotide sequences. If not specified, the translation table used when building the pangenome will be used. This can be accessed using 'ppanggolin info'. | +| `--tmpdir` | str | `/tmp` | Directory for storing temporary files. | +| `--keep_tmp` | bool | False | Keep temporary files (useful for debugging). | +| `-c, --cpu` | int | 1 | Number of available CPUs. | + +#### Common arguments for ppanggolin context + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin msa` + +Compute Multiple Sequence Alignments for pangenome gene families. + +#### Required arguments for ppanggolin msa + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | +| `-o, --output` | Path | — | Output directory where the file(s) will be written. Required: Yes | + +#### Optional arguments. Indicating 'all' writes all elements. Writing a partition ('persistent', 'shell', 'cloud', 'core' or 'accessory') write the elements associated to said partition. for ppanggolin msa + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--soft_core` | float | 0.95 | Soft core threshold to use if 'softcore' partition is chosen | +| `--dup_margin` | float | 0.05 | Minimum ratio of genomes in which the family must have multiple genes for it to be considered 'duplicated' | +| `--single_copy` | bool | False | Use report gene families that are considered 'single copy', for details see option --dup_margin | +| `--partition` | str | `core` | compute Multiple Sequence Alignment of the gene families in the given partition
Choices: `all`, `persistent`, `shell`, `cloud`, `core`, `accessory`, `softcore` | +| `--source` | str | `protein` | indicates whether to use protein or dna sequences to compute the msa
Choices: `dna`, `protein` | +| `--phylo` | bool | False | Writes a whole genome msa file for additional phylogenetic analysis | +| `--use_gene_id` | bool | False | Use gene identifiers rather than genome names for sequences in the family MSA (genome names are used by default) | +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified, the translation table used when building the pangenome will be used. This can be accessed using 'ppanggolin info'. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--tmpdir` | str | `/tmp` | Directory for storing temporary files. | + +#### Common arguments for ppanggolin msa + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +### `ppanggolin projection` + +Annotates external genomes with an existing pangenome. + +#### Required arguments for ppanggolin projection + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-p, --pangenome` | Path | — | Path to the pangenome .h5 file. | +| `--fasta` | Path | — | Specify a FASTA file containing the genomic sequences of the genome(s) you wish to annotate, or provide a tab-separated file listing genome names alongside their respective FASTA file paths, with one line per genome. | +| `--anno` | Path | — | Specify an annotation file in GFF/GBFF format for the genome you wish to annotate. Alternatively, you can provide a tab-separated file listing genome names alongside their respective annotation file paths, with one line per genome. If both an annotation file and a FASTA file are provided, the annotation file will take precedence. | + +#### Single Genome Arguments for ppanggolin projection + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-n, --genome_name` | str | `input_genome` | Specify the name of the genome to annotate when providing a single FASTA or annotation file. | +| `--circular_contigs` | tuple | — | Specify the contigs of the input genome that should be treated as circular when providing a single FASTA or annotation file. | + +#### Optional arguments for ppanggolin projection + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-o, --output` | Path | `ppanggolin_projection__` | Output directory. | +| `--translation_table` | int | 11 | Translation table (genetic code) to use. If not specified, the translation table used when building the pangenome will be used. This can be accessed using 'ppanggolin info'. | +| `--no_defrag` | bool | False | Do not realign gene families to link fragments with their non-fragmented gene family. (Default: False) | +| `--fast` | bool | False | Use representative sequences of gene families for input gene alignment. This option is faster but may be less sensitive. By default, all pangenome genes are used. | +| `--identity` | float | 0.8 | Minimum identity percentage threshold. | +| `--coverage` | float | 0.8 | Minimum coverage percentage threshold. | +| `--use_pseudo` | bool | False | In the context of provided annotation, use this option to read pseudogenes. (Default behavior is to ignore them). | +| `--dup_margin` | float | 0.05 | Minimum ratio of genomes in which the family must have multiple genes for it to be considered 'duplicated'. This metric is used to compute completeness and duplication of the input genomes. | +| `--soft_core` | float | 0.95 | Soft-core threshold used when generating general statistics on the projected genome. This threshold does not influence PPanGGOLiN's partitioning. The value determines the minimum fraction of genomes that must possess a gene family for it to be considered part of the soft core. | +| `--spot_graph` | bool | False | Write the spot graph to a file, with pairs of blocks of single-copy markers flanking RGPs as nodes. This graph can be used to visualize nodes that have RGPs from the input genome. | +| `--graph_formats` | str | `['gexf']` | Format of the output graph.
Choices: `gexf`, `graphml` | +| `--gff` | bool | False | Generate GFF files with projected pangenome annotations for each input genome. | +| `--proksee` | bool | False | Generate JSON map files for PROKSEE with projected pangenome annotations for each input genome. | +| `--table` | bool | False | Generate a TSV file for each input genome with pangenome annotations. | +| `--compress` | bool | False | Compress the files in .gz. | +| `--add_sequences` | bool | False | Include input genome DNA sequences in GFF and Proksee output. | +| `-c, --cpu` | int | 1 | Number of available CPUs. | +| `--tmpdir` | Path | `/tmp` | Directory for storing temporary files. | +| `--keep_tmp` | bool | False | Keep temporary files (useful for debugging). | +| `--add_metadata` | bool | False | Include metadata information in the output files if any have been added to pangenome elements (see ppanggolin metadata command). | +| `--metadata_sources` | str | — | Which source of metadata should be written. By default all metadata sources are included. | +| `--metadata_sep` | str | `|` | The separator used to join multiple metadata values for elements with multiple metadata values from the same source. This character should not appear in metadata values. | + +#### Common arguments for ppanggolin projection + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | + + +## Utility command + +### `ppanggolin utils` + +Helper side commands. + +#### Required arguments for ppanggolin utils + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--default_config` | str | — | Generate a config file with default values for the given subcommand.
Choices: `annotate`, `cluster`, `graph`, `partition`, `rarefaction`, `workflow`, `panrgp`, `panmodule`, `all`, `draw`, `write_pangenome`, `write_genomes`, `write_metadata`, `fasta`, `msa`, `metrics`, `align`, `info`, `rgp`, `spot`, `module`, `context`, `projection`, `rgp_cluster`, `metadata`, `utils` | + +#### Config arguments for ppanggolin utils + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `-o, --output` | Path | `default_config.yaml` | name and path of the config file with default parameters written in yaml. | + +#### Common arguments for ppanggolin utils + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `--verbose` | int | 1 | Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.
Choices: `0`, `1`, `2` | +| `--log` | str | `stdout` | Log output file. | +| `-d, --disable_prog_bar` | bool | False | Disable the progress bars. | +| `-f, --force` | bool | False | Force overwrite in the output directory and pangenome output file. | +| `--config` | Path | — | Specify command arguments through a YAML configuration file. | diff --git a/docs/user/install.md b/docs/user/install.md index 3d75427b..d945b2e7 100644 --- a/docs/user/install.md +++ b/docs/user/install.md @@ -1,7 +1,8 @@ # Installation ```{note} -Supported Python versions are 3.9, 3.10, 3.11 and 3.12 +Supported Python versions are 3.10, 3.11, 3.12 and 3.13. + ``` ## Installing PPanGGOLiN with Conda (Recommended) diff --git a/docs/user/metadata.md b/docs/user/metadata.md index 2a1cbebf..8bad3437 100644 --- a/docs/user/metadata.md +++ b/docs/user/metadata.md @@ -1,6 +1,6 @@ # Metadata and Pangenome -## Associating Metadata to Pangenome Elements +## Associating Metadata with Pangenome Elements The `metadata` command allows the addition of metadata linked to various pangenome elements. Metadata can be associated with genes, genomes, families, RGPs, spots, and modules using a simple TSV file. @@ -25,7 +25,7 @@ Certain information (such as species, strain, and dbx_ref) is extracted from gen ### Metadata Format -PPanGGOLiN offers a highly flexible metadata file format. Only one column name is mandatory, and it aligns with the assignment argument chosen by the user (ie. families, RGPS...). +PPanGGOLiN offers a highly flexible metadata file format. Only one column name is mandatory, and it aligns with the assignment argument chosen by the user (*i.e.* families, RGPS...). For instance, the TSV file used to assign metadata to gene families for functional annotation might resemble the following: @@ -41,6 +41,67 @@ For instance, the TSV file used to assign metadata to gene families for function As you can see in the above table, one element (here GF_2) can be associated with with multiple metadata entries. ``` +The file has to respect the following rules: + +- it is tab-separated and has a header line; +- it contains at least two columns, one of them named after the `--assign` value; +- every column name is a valid identifier, that is, it only contains letters, digits and underscores, and does not start with a digit. Column names such as `E-value`, `hmm acc` or `target name` are rejected and have to be renamed; +- a cell containing a single `-` is interpreted as a missing value. + +### Example: annotating gene families with Pfam + +The following example shows the complete process of annotating the gene families of a pangenome with [Pfam](https://www.ebi.ac.uk/interpro/), and adding the result to the pangenome. + +**1. Export the representative sequences of the gene families** + +Each gene family of the pangenome is represented by one sequence, which can be written with the `fasta` command (see [how to write Gene families sequences](./writeFasta.md#gene-families) for more details): + +```bash +ppanggolin fasta -p pangenome.h5 --output MY_PROT --prot_families all +``` + +This creates the file `MY_PROT/all_protein_families.faa`, containing one protein sequence per gene family. The identifier of each sequence is the identifier of the gene family it represents, which is precisely what has to be given back to PPanGGOLiN in the next steps. + +**2. Annotate the sequences with Pfam** + +The resulting FASTA file can then be given to any annotation tool. For Pfam, this is typically done with [HMMER](http://hmmer.org/) against the `Pfam-A.hmm` database, or with the `pfam_scan.pl` script distributed by Pfam: + +```bash +hmmsearch --cut_ga --tblout pfam_hits.txt Pfam-A.hmm MY_PROT/all_protein_families.faa +``` + +**3. Format the result as a PPanGGOLiN metadata file** + +The output of the annotation tool has to be turned into a TSV file respecting the rules given above. In practice this means keeping the columns of interest, renaming the column holding the gene family identifiers to `families`, and renaming any column whose name is not a valid identifier. + +For instance, an annotation result looks like this: + +| query_name | accession | target_name | E-value | score | description_of_target | +|------------|------------|-------------|---------|-------|-------------------------------| +| GF_1 | PF00005.30 | ABC_tran | 1.2e-45 | 152.3 | ABC transporter | +| GF_2 | PF07690.19 | MFS1 | 3.4e-30 | 101.7 | Major facilitator superfamily | + +becomes: + +| families | pfam_name | pfam_accession | e_value | score | description | +|----------|-----------|----------------|---------|--------|-------------------------------| +| GF_1 | ABC_tran | PF00005.30 | 1.2e-45 | 152.3 | ABC transporter | +| GF_2 | MFS_1 | PF07690.19 | 3.4e-30 | 101.7 | Major facilitator superfamily | + +Only the column names change: the `query_name` column, which holds the gene family identifiers written at step 1, is renamed to `families`, and `E-value` is renamed to `e_value` so that it is a valid identifier. The remaining columns are free, both in number and in name. + +```{note} +The order of the columns does not matter. PPanGGOLiN identifies the columns by their name, so the `families` column can be anywhere in the file. +``` + +**4. Add the annotations to the pangenome** + +```bash +ppanggolin metadata -p pangenome.h5 --metadata pfam_families.tsv --source pfam --assign families +``` + +The annotations are now stored in the pangenome under the source `pfam`, and are propagated to the outputs described above. A gene family matched by several Pfam entries simply appears on several lines, as shown in the previous section. + ### Command Specific Option Details #### `--metadata` diff --git a/docs/user/projection.md b/docs/user/projection.md index 8dda6bbc..72f85e0a 100644 --- a/docs/user/projection.md +++ b/docs/user/projection.md @@ -2,46 +2,62 @@ The ppanggolin projection command allows you to annotate external genomes using an existing pangenome. This process eliminates the need to recompute all components, streamlining the annotation process. Input genomes are expected to belong to the same species. -Genes within the input genome are aligned with genes in the pangenome to determine their gene families and partitions. Genes that do not align with any existing gene in the pangenome are considered specific to the input genome and are assigned to the "Cloud" partition. The number of this specific cloud families are detailed in the summary table. +Genes within the input genome are aligned with genes in the pangenome to determine their gene families and partitions. Genes that do not align with any existing gene in the pangenome are considered specific to the input genome and are assigned to the "Cloud" partition. The numbers of these specific cloud families are detailed in the summary table. Based on the alignment and partition assignment, Regions of Plasticity (RGPs) within the input genome are predicted. Each RGP that is not located on a contig border is assigned to a spot of insertion. Finally, conserved modules of the pangenome found in the input genome are reported in the output files. -## Input files: -This command supports two input modes depending on whether you want to project a single genome or multiple genomes at once: +### Input files -Multiple Files in One TSV: -- **Options**: `--fasta` or `--anno` -- **Description**: You can provide a tab-separated file listing genome names alongside their respective FASTA genomic sequences or annotation filepaths, with one line per genome. This mode is suitable when you want to annotate multiple genomes in a single operation. The format of this file is identical to the format used in the annotate and workflow commands; for more details, refer [here](./PangenomeAnalyses/pangenomeAnnotation.md). +Projection accepts the same genome input formats as the `annotate` and `all` commands. Genomes can be provided either as genomic FASTA files using `--fasta` or as GFF3/GBFF annotation files using `--anno`. +You can project either a single genome or multiple genomes at once. + +### Project multiple genomes + +To project multiple genomes, provide `--fasta` or `--anno` with a tab-separated file listing one genome per line. The file format is identical to the input files used to create a pangenome. + +Examples with 50 *Chlamydia trachomatis* genomes are available in the [testingDataset](https://github.com/labgem/PPanGGOLiN/tree/master/testingDataset) directory: [`genomes.gbff.list`](https://github.com/labgem/PPanGGOLiN/blob/master/testingDataset/genomes.gbff.list) for annotation files and [`genomes.fasta.list`](https://github.com/labgem/PPanGGOLiN/blob/master/testingDataset/genomes.fasta.list) for FASTA files. -**Example Usage:** -Suppose you have a TSV file named `external_genome_paths.txt` listing multiple annotation files of external genomes. To execute the projection, use the following command: ```bash -ppanggolin projection -p pangenome.h5 --anno external_genome_paths.txt +ppanggolin projection -p pangenome.h5 --anno genomes.gbff.list ``` +or: + +```bash +ppanggolin projection -p pangenome.h5 --fasta genomes.fasta.list +``` +### Project a single genome -Single File: -- **Options**: `--genome_name` with `--fasta` or `--anno` and `--circular_contigs` (optional) -- **Description**: When annotating a single genome, you can directly provide a single FASTA genomic sequence file or an annotation file in GFF/GBFF format. Additionally, specify the name of the genome using the `--genome_name` option. You can also indicate circular contigs using the `--circular_contigs` option when necessary. +For a single genome, provide the file directly and specify its name with `--genome_name`. The input can be either a genomic FASTA file or a GFF3/GBFF annotation file. -**Example Usage:** -Suppose you have a single fasta file named `genome_A.fasta` of an external genome. To execute the projection, use the following command: ```bash -ppanggolin projection -p pangenome.h5 --anno genome_A.fasta --genome_name genome_A +ppanggolin projection -p pangenome.h5 \ + --fasta genome_A.fasta \ + --genome_name genome_A ``` +or: + +```bash +ppanggolin projection -p pangenome.h5 \ + --anno genome_A.gbff \ + --genome_name genome_A +``` + +The optional `--circular_contigs` argument can be used to specify circular contigs. + ## Output Files Within the Output directory, the `summary_projection.tsv` file provides an overview of the projection, featuring one line per genome. This file includes all the columns described in the [genome-statistics table](./PangenomeAnalyses/pangenomeStat.md#genome-statistics-table) section, along with specific projection-related columns detailed below: -| Column | Description | -|-----------------------------|-----------------------------------------------------------------------------------------------| -| Cloud_specific_families | Number of cloud-specific families. These gene families do not match any existing families within the pangenome. | -| Pangenome_file | The file path of the pangenome used in the projection. | -| New_spots | Number of newly identified spots in the input genome. | +| Column | Description | +|-------------------------|-----------------------------------------------------------------------------------------------------------------| +| Cloud_specific_families | Number of cloud-specific families. These gene families do not match any existing families within the pangenome. | +| Pangenome_file | The file path of the pangenome used in the projection. | +| New_spots | Number of newly identified spots in the input genome. | @@ -51,9 +67,9 @@ Additionally, within the Output directory, there is a subdirectory for each inpu For Gene Family and Partition of Input Genes: - `cds_sequences.fasta`: This file contains the sequences of coding regions (CDS) from the input genome. -- `gene_to_gene_family.tsv`: It provides the mapping of genes to gene families of the pangenome. Its format follows [this output](PangenomeAnalyses/pangenomeAnalyses.md#gene-families-to-genes-associations) +- `gene_to_gene_family.tsv`: It provides the mapping of genes to gene families of the pangenome. Its format follows [this output](PangenomeAnalyses/pangenomeStat.md#gene-families-to-gene-associations). - `sequences_partition_projection.tsv`: This file maps the input genes to its partition (Persistent, Shell or Cloud). -- `specific_genes.tsv`: This file lists the gene of the input genomes that do not align to any gene of the pangenome. These genes are assigned to Cloud parititon. +- `specific_genes.tsv`: This file lists the gene of the input genomes that do not align to any gene of the pangenome. These genes are assigned to Cloud partition. For RGPs and Spots: diff --git a/docs/user/whatsNewInV2.md b/docs/user/whatsNewInV2.md new file mode 100644 index 00000000..042949ad --- /dev/null +++ b/docs/user/whatsNewInV2.md @@ -0,0 +1,59 @@ +# What's new in PPanGGOLiN v2 + +PPanGGOLiN v2 is the result of the work carried out since mid-2021, released under version numbers `2.0.0` and above from January 2024 onwards. This page summarizes what changed with respect to PPanGGOLiN as originally published ([Gautreau et al. 2020](https://doi.org/10.1371/journal.pcbi.1007732)), as well as panRGP ([Bazin et al. 2020](https://doi.org/10.1093/bioinformatics/btaa792)) and panModule ([Bazin et al. 2021](https://doi.org/10.1101/2021.12.06.471380)), taking release `1.1.136` (February 2021) as the reference point. It is intended for users coming from a v1 release who want to know what is new and what has moved. + +The features listed below were released progressively, and some of them are already available in the later `1.2.x` releases. The major version number was raised to `2.0.0` at the point where the redesigned pangenome file broke compatibility with the HDF5 files produced by v1, marking the boundary between the two versions. The rationale behind these changes is described in the PPanGGOLiN v2 publication. + +## New analyses + +| Command | Description | Documentation | +|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------| +| `projection` | Annotate external genomes using an already computed pangenome, without recomputing it. Genes are assigned to existing gene families and partitions, then RGPs, spots and modules are predicted for the projected genome. | [Projection](projection.md) | +| `context` | Extract conserved genomic neighborhoods around genes of interest, using the neighborhood relationships already encoded in the pangenome graph. | [Genomic context](genomicContext.md) | +| `rgp_cluster` | Cluster RGPs across genomes based on their shared gene families, using a Gene Repertoire Relatedness (GRR) score. | [RGP clustering](./RGP/rgpAnalyses.md#rgp-clustering) | +| `metadata` | Attach arbitrary metadata to any pangenome element (genes, contigs, genomes, gene families, edges, RGPs, spots and modules) from a tabulated file. Metadata is stored in the pangenome file and propagated to the outputs. | [Metadata](metadata.md) | + +## New utility commands + +| Command | Description | Documentation | +|------------------|------------------------------------------------------------------------------------|------------------------------------------------------------------| +| `metrics` | Compute pangenome metrics such as genomic fluidity. | [Pangenome metrics](./PangenomeAnalyses/pangenomeAnalyses.md#pangenome-metrics) | +| `write_metadata` | Export the metadata stored in a pangenome. | [Metadata](metadata.md) | +| `utils` | Generate a default YAML configuration file with `--default_config`. | [Configuration file](practicalInformation.md#configuration-file) | + +## Reorganized commands + +The `write` command has been split into three commands with clearer scopes: + +| v1 | v2 | +|------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| +| `ppanggolin write --regions --spots ...` | `ppanggolin write_pangenome` — pangenome-level outputs (gene families, partitions, RGPs, spots, modules, graphs, statistics) | +| `ppanggolin write --projection` | `ppanggolin write_genomes --table` — genome-level outputs, one file per genome | +| — | `ppanggolin write_metadata` — metadata outputs | + + +## New output formats + +- **Proksee** (`write_genomes --proksee`, and the `projection` outputs): JSON files that can be loaded in [Proksee](https://proksee.ca/) to explore circular genome maps annotated with pangenome information. +- **GFF3** (`write_genomes --gff`): genome annotations enriched with pangenome information (gene family, partition, RGP, spot, module). +- **graph-tool** (`write_pangenome --gt`): the pangenome graph in the compressed `gt` format, in addition to the GEXF and JSON formats already available in v1. +- **RGP gene families** (`write_pangenome --regions_families`): the gene family composition of each RGP. + +## Usability + +- **Configuration files.** Every command accepts a `--config` option pointing to a YAML file, so that parameters no longer have to be passed on the command line. This is particularly useful for the workflow commands, which run several steps in a single call, and for integration into computational platforms. A template can be generated with `ppanggolin utils --default_config`, and the format is described in the [Configuration file](practicalInformation.md#configuration-file) section. +- **Parameter tracking.** Parameters are resolved consistently (command line, then configuration file, then defaults), checked for consistency across analysis steps, and stored inside the pangenome file, so that an analysis can be inspected and reproduced afterward with `ppanggolin info --parameters`. +- **Defragmentation is now enabled by default.** The v1 opt-in `--defrag` option of the `cluster` command has been replaced by the opt-out `--no_defrag` option. +- **Documentation.** The documentation has been fully rewritten and is now hosted on [ReadTheDocs](https://ppanggolin.readthedocs.io), including a reference of all commands and their options and an API reference for programmatic use. +- **Distribution.** PPanGGOLiN v2 is available on [Bioconda](https://anaconda.org/bioconda/ppanggolin) and [PyPI](https://pypi.org/project/PPanGGOLiN/). + +## Performance and file format + +- **Pyrodigal replaces Prodigal.** Gene prediction now uses [Pyrodigal](https://github.com/althonos/pyrodigal), which removes the input/output overhead of calling an external binary during annotation. +- **Redesigned HDF5 structure.** The internal layout of the pangenome file was reworked, substantially reducing file size and giving faster and more convenient programmatic access. +- **Lower memory usage when writing sequences.** Sequences are read directly from the HDF5 file instead of being loaded into memory beforehand. + +## Compatibility with v1 pangenome files + +The redesign of the HDF5 structure is what defines the boundary between v1 and v2: the major version number was raised to `2.0.0` precisely because pangenome files became incompatible at that point. Pangenome files produced with v1 therefore cannot be read by v2. PPanGGOLiN checks the version stored in the file and raises an explicit error if it was created by a v1 release, so such pangenomes have to be recomputed with v2. + diff --git a/ppanggolin/RGP/genomicIsland.py b/ppanggolin/RGP/genomicIsland.py index 06ce418d..20572b44 100644 --- a/ppanggolin/RGP/genomicIsland.py +++ b/ppanggolin/RGP/genomicIsland.py @@ -412,6 +412,10 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser :return : parser arguments for align command """ parser = sub_parser.add_parser("rgp", formatter_class=argparse.RawTextHelpFormatter) + parser.description = ( + "Predicts Regions of Genomic Plasticity in the genomes of your pangenome." + ) + parser.category = "Regions of Genomic Plasticity" parser_rgp(parser) return parser @@ -427,7 +431,11 @@ def parser_rgp(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) optional = parser.add_argument_group(title="Optional arguments") diff --git a/ppanggolin/RGP/rgp_cluster.py b/ppanggolin/RGP/rgp_cluster.py index 8605aaba..83ce3a82 100644 --- a/ppanggolin/RGP/rgp_cluster.py +++ b/ppanggolin/RGP/rgp_cluster.py @@ -1,824 +1,1412 @@ #!/usr/bin/env python3 # default libraries -import logging import argparse -import os -from itertools import combinations -from collections.abc import Callable +import json +import logging +import typing as tp from collections import defaultdict -from typing import Dict, List, Tuple, Set, Union, Any +from collections.abc import Callable +from dataclasses import dataclass, field +from itertools import combinations from pathlib import Path +from typing import Dict, List, Set, Union -# installed libraries -from tqdm import tqdm import networkx as nx import pandas as pd +from pyroaring import BitMap -# local libraries -from ppanggolin.pangenome import Pangenome -from ppanggolin.region import Region, Spot, Module -from ppanggolin.formats import check_pangenome_info -from ppanggolin.utils import restricted_float, mk_outdir +# installed libraries +from tqdm import tqdm + +from ppanggolin.formats.h5reader import H5Reader, TableAttribute from ppanggolin.geneFamily import GeneFamily +# local libraries +from ppanggolin.utils import mk_outdir, restricted_float -class IdenticalRegions: - """ - Represents a group of Identical Regions within a pangenome. - :param name: The name of the identical region group. - :param identical_rgps: A set of Region objects representing the identical regions. - :param families: A set of GeneFamily objects associated with the identical regions. - :param is_contig_border: A boolean indicating if the identical regions span across contig borders. - """ +class RegionProxy: + """Represent a single RGP or a merged group of identical RGPs in the clustering graph.""" def __init__( self, + ID: int, name: str, - identical_rgps: Set[Region], - families: Set[GeneFamily], + families: BitMap, is_contig_border: bool, + is_whole_contig: bool, + children=None, + modules=None, + contig=None, + organism=None, + length=0, ): - if not isinstance(identical_rgps, set): - raise TypeError("Expected 'identical_rgps' to be a set") - else: - if len(identical_rgps) == 0: - raise ValueError("Set of identical_rgps must not be empty") - if not all(isinstance(region, Region) for region in identical_rgps): - raise TypeError("All element in identical_rgps must be `Region`") - if not isinstance(families, set): - raise TypeError("Expected 'families' to be a set") + """Initialize the proxy with the RGP identity and family content. + + :param ID: Stable identifier used in the graph and output tables. + :param name: Name of the region or identical-group label. + :param families: BitMap of family identifiers present in the RGP. + :param is_contig_border: Whether the region touches a contig border. + :param is_whole_contig: Whether the region spans the whole contig. + :param children: Child RGP proxies for merged identical regions. + :param modules: Module identifiers associated with the RGP. + :param contig: Contig name associated with the region. + :param organism: Organism name associated with the region. + :param length: Number of genes in the region. + """ + self.ID: int = ID + self.name: str = name + self.families: BitMap = families + self.children: set[RegionProxy] = children + self.modules: set[int] = set(modules) if modules is not None else set() + + if self.children: + self.organism: str = next(iter(self.children)).organism + self.contig: str = next(iter(self.children)).contig else: - if len(families) == 0: - raise ValueError("Set of families must not be empty") - if not all(isinstance(family, GeneFamily) for family in families): - raise TypeError("All element in families must be `GeneFamilies`") - self.name = name - self.families = families - self.rgps = identical_rgps - self.is_contig_border = is_contig_border - self.ID = Region.id_counter - - Region.id_counter += 1 - - def __eq__(self, other: "IdenticalRegions") -> bool: - """ - Check if two IdenticalRegions objects are equal based on their families, - identical regions, and contig border status. - - :param other: The IdenticalRegions object to compare. - :return: True if the objects are equal, False otherwise. - """ - if not isinstance(other, IdenticalRegions): - # don't attempt to compare against unrelated types - raise TypeError( - "'IdenticalRegions' type object was expected, " - f"but '{type(other)}' type object was provided." - ) + self.organism: str = organism + self.contig: str = contig - return ( - self.families == other.families - and self.rgps == other.rgps - and self.is_contig_border == other.is_contig_border - ) + self.length: int = length + self.nb_families: int = len(self.families) + + self.is_contig_border: bool = is_contig_border + self.is_whole_contig: bool = is_whole_contig + + @property + def is_identical_region(self) -> bool: + """Whether this region proxy represents a merged group of identical RGPs. + + :return: True when the proxy stores multiple child regions. + """ + return self.children is not None and len(self.children) > 0 def __repr__(self): - return ( - f"IdenticalRegions(name='{self.name}', num_rgps={len(self.rgps)}, num_families={len(self.families)}," - f" is_contig_border={self.is_contig_border})" - ) + """Return a compact representation of the proxy for debugging. + + :return: Debug-friendly representation of the region proxy. + """ + return f"RegionProxy2(ID={self.ID}, name='{self.name}')" def __str__(self): + """Return the region name. + + :return: Human-readable name of the region. + """ return self.name - def __hash__(self): - return id(self) + def __hash__(self) -> int: + """Hash the proxy using a stable region identifier. + + :return: Hash value derived from the region ID. + """ + # Hash on the stable, deterministically-assigned ID rather than id(self): + # object addresses vary between runs and make set/dict iteration order + # (and therefore graph construction order) non-reproducible. + return hash(self.ID) + + def __eq__(self, rhs: "RegionProxy") -> bool: + """Check whether two region proxies describe the same content. + + :param rhs: Other region proxy to compare against. + :return: Whether the two proxies are equivalent. + """ + return ( + self.families == rhs.families + and self.children == rhs.children + and self.is_contig_border == rhs.is_contig_border + ) def __lt__(self, obj): + """Compare regions by their stable identifier. + + :param obj: Other region proxy. + :return: Whether this region comes before the other one. + """ return self.ID < obj.ID def __gt__(self, obj): + """Compare regions by their stable identifier. + + :param obj: Other region proxy. + :return: Whether this region comes after the other one. + """ return self.ID > obj.ID def __le__(self, obj): + """Compare regions by their stable identifier. + + :param obj: Other region proxy. + :return: Whether this region is less than or equal to the other one. + """ return self.ID <= obj.ID def __ge__(self, obj): - return self.ID >= obj.ID + """Compare regions by their stable identifier. - @property - def genes(self): - """ - Return iterable of genes from all RGPs that are identical in families + :param obj: Other region proxy. + :return: Whether this region is greater than or equal to the other one. """ - for rgp in self.rgps: - yield from rgp.genes + return self.ID >= obj.ID - @property - def spots(self) -> Set[Spot]: - """ - Return spots from all RGPs that are identical in families - """ - spots = {rgp.spot for rgp in self.rgps if rgp.spot is not None} - return spots - @property - def modules(self) -> Set[Module]: - """ - Return iterable of genes from all RGPs that are identical in families - """ - modules = set() - for rgp in self.rgps: - modules |= rgp.modules +@dataclass +class RGPTable: + """Schema for the RGP membership table stored in the HDF5 file.""" - return modules + rgp: tp.Annotated[ + str, TableAttribute(name="RGP", transform=lambda x: x.decode("utf-8")) + ] + gene: tp.Annotated[ + str, TableAttribute(name="gene", transform=lambda x: x.decode("utf-8")) + ] + _table: str = "/RGP" -def compute_grr( - rgp_a_families: Set[GeneFamily], rgp_b_families: Set[GeneFamily], mode: Callable -) -> float: - """ - Compute gene repertoire relatedness (GRR) between two rgp. - Mode can be the function min to compute min GRR or max to compute max_grr +@dataclass +class GeneFamTable: + """Schema for the gene-to-family table stored in the HDF5 file.""" - :param rgp_a_families: Rgp A - :param rgp_b_families: rgp B - :param mode: min or max function + gene: tp.Annotated[ + str, TableAttribute(name="gene", transform=lambda x: x.decode("utf-8")) + ] + family: tp.Annotated[ + str, TableAttribute(name="geneFam", transform=lambda x: x.decode("utf-8")) + ] + _table: str = "/geneFamilies" - :return: GRR value between 0 and 1 - """ - grr = len(rgp_a_families & rgp_b_families) / mode( - len(rgp_a_families), len(rgp_b_families) - ) +@dataclass +class AnnotationsGeneTable: + """Schema for the annotations gene table used to map genes to contigs.""" - return grr + name: tp.Annotated[ + str, TableAttribute(name="ID", transform=lambda x: x.decode("utf-8")) + ] + contig: tp.Annotated[int, TableAttribute(name="contig")] + _table: str = "/annotations/genes" -def compute_jaccard_index(rgp_a_families: set, rgp_b_families: set) -> float: - """ - Compute jaccard index between two rgp based on their families. +@dataclass +class RGPSpotTable: + """Schema for the RGP-to-spot assignment table.""" - :param rgp_a_families: Rgp A - :param rgp_b_families: rgp B + rgp: tp.Annotated[ + str, TableAttribute(name="RGP", transform=lambda x: x.decode("utf-8")) + ] + spot: tp.Annotated[int, TableAttribute(name="spot")] + _table: str = "/spots" - :return : Jaccard index - """ - jaccard_index = len(rgp_a_families & rgp_b_families) / len( - rgp_a_families | rgp_b_families - ) +@dataclass +class ModuleTable: + """Schema for the family-to-module membership table.""" - return jaccard_index + fam: tp.Annotated[ + str, TableAttribute(name="geneFam", transform=lambda x: x.decode("utf-8")) + ] + module: tp.Annotated[int, TableAttribute(name="module")] + _table: str = "/modules" -def add_info_to_rgp_nodes(graph, regions: List[Region], region_to_spot: dict): - """ - Format RGP information into a dictionary for adding to the graph. +@dataclass +class ContigTable: + """Schema for contig metadata stored in the HDF5 pangenome.""" - This function takes a list of RGPs and a dictionary mapping each RGP to its corresponding spot ID, - and formats the RGP information into a dictionary for further processing or addition to a graph. + genome: tp.Annotated[ + str, TableAttribute(name="genome", transform=lambda x: x.decode("utf-8")) + ] + contig: tp.Annotated[ + str, TableAttribute(name="name", transform=lambda x: x.decode("utf-8")) + ] + is_circular: tp.Annotated[bool, TableAttribute(name="is_circular")] + idx: tp.Annotated[int, TableAttribute(name="ID")] + _table: str = "/annotations/contigs" - :param graph: RGPs graph - :param regions: A list of RGPs. - :param region_to_spot: A dictionary mapping each RGP to its corresponding spot ID. - :return: A dictionary with RGP id as the key and a dictionary containing information on the corresponding RGP as value. - """ - region_attributes = {} - for region in regions: - region_info = { - "contig": region.contig.name, - "genome": region.organism.name, - "name": region.name, - "genes_count": len(region), - "is_contig_border": region.is_contig_border, - "is_whole_contig": region.is_whole_contig, - "spot_id": get_spot_id(region, region_to_spot), - "modules": ";".join({str(module) for module in region.modules}), - "families_count": region.number_of_families, - } +@dataclass +class GenesTable: + """Schema for the gene annotations table used to resolve contig membership.""" - region_attributes[region.ID] = region_info + name: tp.Annotated[ + str, TableAttribute(name="ID", transform=lambda x: x.decode("utf-8")) + ] + genedata: tp.Annotated[int, TableAttribute(name="genedata_id")] + contig: tp.Annotated[int, TableAttribute(name="contig")] + _table: str = "/annotations/genes" - node_attributes = graph.nodes[region.ID] - node_attributes.update(region_info) - return region_attributes +@dataclass +class GeneDataTable: + """Schema for the per-gene coordinate table used in contig border logic.""" + # gene_type: tp.Annotated[str, TableAttribute(name="gene_type", transform=lambda x: x.decode('utf-8'))] + idx: tp.Annotated[int, TableAttribute(name="genedata_id")] + start: tp.Annotated[int, TableAttribute(name="start")] + stop: tp.Annotated[int, TableAttribute(name="stop")] + position: tp.Annotated[int, TableAttribute(name="position")] + _table: str = "/annotations/genedata" -def join_dicts(dicts: List[Dict[str, Any]], delimiter: str = ";") -> Dict[str, Any]: - """ - Join dictionaries by concatenating the values with a custom delimiter for common keys. - Given a list of dictionaries, this function creates a new dictionary where the values for common keys - are concatenated with the specified delimiter. +@dataclass(slots=True) +class RGPGeneProxy: + """Store the coordinate information of a gene within an RGP.""" - :param dicts: A list of dictionaries to be joined. - :param delimiter: The delimiter to use for joining values. Default is ';'. - :return: A dictionary with joined values for common keys. - """ - final_dict = defaultdict(list) - for dict_obj in dicts: - for k, v in dict_obj.items(): - final_dict[k].append(str(v)) - return {k: delimiter.join(v) for k, v in final_dict.items()} + start: int + stop: int + position: int -def format_rgp_metadata(rgp: Region) -> Dict[str, str]: - """ - Format RGP metadata by combining source and field values. +@dataclass(slots=True) +class RGPGenes: + """Bundle contig-level information for a set of genes belonging to an RGP.""" - Given an RGP object with metadata, this function creates a new dictionary where the keys - are formatted as 'source_field' and the values are concatenated with '|' as the delimiter. + contig: int + is_circular_contig: bool + genes: list[RGPGeneProxy] - :param rgp: The RGP object with metadata. - :return: A dictionary with formatted metadata. - """ - source_field_2_value = defaultdict(list) - for rgp_metadata in rgp.metadata: - source = rgp_metadata.source - for field in rgp_metadata.fields: - source_field_2_value[f"{source}_{field}"].append( - str(rgp_metadata.get(field)) - ) - return { - col_name: "|".join(values) for col_name, values in source_field_2_value.items() - } +@dataclass(slots=True) +class RGPInfo: + """In-memory summary of an RGP before building graph nodes.""" + name: str + families: set[str] + families_ids: BitMap + is_contig_border: bool + is_whole_contig: bool + contig: str -def add_rgp_metadata_to_graph( - graph: nx.Graph, rgps: List[Union[Region, IdenticalRegions]] -) -> None: - """ - Add metadata from Region or IdenticalRegions objects to the graph. - :param graph: The graph to which the metadata will be added. - :param rgps: A set of Region or IdenticalRegions objects containing the metadata to be added. +@dataclass(slots=True) +class ContigBorderPosition: + """Store the first and last gene coordinates on a contig.""" - """ - for rgp in rgps: - element_to_metadata_sources = { - "family": set(), - "gene": set(), - "module": set(), - "spot": set(), - } + last_gene_position: int + last_gene_idx: int + first_gene_idx: int + first_gene_position: int + gene_count: int - for family in rgp.families: - element_to_metadata_sources["family"] |= { - metadata.source for metadata in family.metadata - } - if family.module: - element_to_metadata_sources["module"] |= { - metadata.source for metadata in family.module.metadata - } - for gene in rgp.genes: - element_to_metadata_sources["gene"] |= { - metadata.source for metadata in gene.metadata - } +@dataclass(slots=True) +class ContigBorderGenes: + """Store the first and last gene names for a contig used in border checks.""" - if isinstance(rgp, Region): - rgp_metadata = rgp.formatted_metadata_dict_to_string() - if rgp.spot is not None: - element_to_metadata_sources["spot"] = { - metadata.source for metadata in rgp.spot.metadata - } + first_gene: str + last_gene: str + gene_count: int - elif isinstance(rgp, IdenticalRegions): - rgp_metadata_dicts = [ - ident_rgp.formatted_metadata_dict_to_string() for ident_rgp in rgp.rgps - ] - rgp_metadata = join_dicts(rgp_metadata_dicts) - element_to_metadata_sources["spot"] |= { - metadata.source for spot in rgp.spots for metadata in spot.metadata - } +@dataclass(slots=False) +class RGPMetric: + """Store the computed similarity metric between two regions.""" - else: - raise TypeError( - f"Expect Region or IdenticalRegions object, not {type(rgp)}" - ) + max_grr: float + min_grr: float + incomplete_aware_grr: float + shared_family: int - for element, metadata_sources in element_to_metadata_sources.items(): - for source in metadata_sources: - graph.nodes[rgp.ID][f"has_{element}_with_{source}"] = True - for metadata_name, value in rgp_metadata.items(): - graph.nodes[rgp.ID][metadata_name] = value +@dataclass(slots=True) +class Contig: + """Basic metadata describing a contig referenced by the clustering analysis.""" + organism: str + is_circular: bool + idx: int + name: str -def add_info_to_identical_rgps( - rgp_graph: nx.Graph, - identical_rgps_objects: List[IdenticalRegions], - rgp_to_spot: Dict[Region, int], -): - """ - Add identical rgps info in the graph as node attributes. - :params rgp_graph: Graph with rgp id as node and grr value as edges - :params rgp_to_identical_rgps: dict with uniq RGP as the key and set of identical rgps as value - """ +RGPMetricType = tp.Literal["max_grr", "min_grr", "incomplete_aware_grr"] - for identical_rgp_obj in identical_rgps_objects: - spots_of_identical_rgp_obj = { - get_spot_id(i_rgp, rgp_to_spot) for i_rgp in identical_rgp_obj.rgps - } - rgp_graph.add_node( - identical_rgp_obj.ID, - identical_rgp_group=True, - name=identical_rgp_obj.name, - families_count=len(identical_rgp_obj.families), - identical_rgp_count=len(identical_rgp_obj.rgps), - identical_rgp_names=";".join( - i_rgp.name for i_rgp in identical_rgp_obj.rgps - ), - identical_rgp_genomes=";".join( - {i_rgp.organism.name for i_rgp in identical_rgp_obj.rgps} - ), - identical_rgp_contig_border_count=len( - [True for i_rgp in identical_rgp_obj.rgps if i_rgp.is_contig_border] - ), - identical_rgp_whole_contig_count=len( - [True for i_rgp in identical_rgp_obj.rgps if i_rgp.is_whole_contig] - ), - identical_rgp_spots=";".join(spots_of_identical_rgp_obj), - spot_id=( - spots_of_identical_rgp_obj.pop() - if len(spots_of_identical_rgp_obj) == 1 - else "Multiple spots" - ), - modules=";".join({str(module) for module in identical_rgp_obj.modules}), +@dataclass +class RGPClusteringOptions: + """Configuration values controlling the RGP clustering workflow.""" + + grr_cutoff: float = 0.3 + metric: RGPMetricType = "incomplete_aware_grr" + unmerge_identical_rgps: bool = True + output: Path = Path("rgp_clustering") + basename: str = "rgp_cluster" + graph_formats: list[str] = ("gexf", "graphml") + with_metadata: bool = False + metadata_sources: list[str] = field(default_factory=list) + ignore_incomplete_rgp: bool = False + disable_prog_bar: bool = False + + +class RGPClustering: + """Load RGP data from an HDF5 pangenome and cluster regions by family content.""" + + def __init__(self, pangenome_h5: Path): + """Initialize an RGP clustering analysis for a pangenome file. + + :param pangenome_h5: Path to the pangenome HDF5 file. + """ + self.h5 = Path(pangenome_h5) + self.rgps: set[RegionProxy] = set() + self.metrics: list[RGPMetric] = [] + self.identical_regions: int = 0 + self.graph: nx.Graph = nx.Graph() + self._rgp_to_spot: dict[str, int] = None + self._fam_to_modules: dict[str, set[str]] = None + self._contig_to_organism: dict[str, str] = None + self._rgp_to_nb_genes: dict[str, int] = None + self._rgp_to_contig_info: dict[str, RGPGenes] = None + self._rgp_to_genes: dict[str, set[str]] = None + self._family_id_to_name: dict[int, str] = None + self._has_metadata: bool = False + self._rgp_metadata_values: Dict[str, Dict[str, list]] = defaultdict( + lambda: defaultdict(list) ) + self._family_metadata_sources: Dict[str, set] = defaultdict(set) + self._gene_metadata_sources: Dict[str, set] = defaultdict(set) + self._spot_metadata_sources: Dict[int, set] = defaultdict(set) + self._module_metadata_sources: Dict[int, set] = defaultdict(set) + def _get_rgp_spot(self, reader: H5Reader) -> dict[str, int]: + """Load the mapping between each RGP and its associated spot. -def add_edges_to_identical_rgps( - rgp_graph: nx.Graph, identical_rgps_objects: List[IdenticalRegions] -): - """ - Replace identical rgp objects by all identical RGPs it contains. + :param reader: H5Reader opened on the pangenome file. + :return: Dictionary mapping every RGP name to its spot identifier. + """ + rgp_to_spot: dict[str, int] = {} + for table in reader.fetch(RGPSpotTable): + rgp_to_spot[table.rgp] = table.spot + return rgp_to_spot - :param rgp_graph: The RGP graph to add edges to. - :param identical_rgps_objects: A dictionary mapping RGPs to sets of identical RGPs. - """ + def _get_fam_to_modules(self, reader: H5Reader) -> dict[str, set[str]]: + """Load the mapping from each gene family to the modules containing it. - identical_edge_data = { - "grr": 1.0, - "max_grr": 1.0, - "min_grr": 1.0, - "identical_famillies": True, - } + :param reader: H5Reader opened on the pangenome file. + :return: Dictionary from family name to set of module identifiers. + """ + fam_to_modules: dict[str, set[str]] = defaultdict(set) + for table in reader.fetch(ModuleTable): + fam_to_modules[table.fam].add(table.module) + return fam_to_modules - added_identical_rgps = [] + def _get_contig_to_organism(self, reader: H5Reader) -> dict[str, str]: + """Map contig names to their associated genome names. - for identical_rgp_obj in identical_rgps_objects: + :param reader: H5Reader opened on the pangenome file. + :return: Dictionary mapping each contig name to its organism name. + """ + contig_to_organism: dict[str, str] = {} + for table in reader.fetch(ContigTable): + contig_to_organism[table.contig] = table.genome + return contig_to_organism + + def _get_contig_to_is_circular(self, reader: H5Reader) -> dict[str, bool]: + """Map contig identifiers to their circularity metadata. + + :param reader: H5Reader opened on the pangenome file. + :return: Dictionary mapping each contig identifier to whether it is circular. + """ + circular_contig_ids: dict[str, bool] = {} + for table in reader.fetch(ContigTable): + circular_contig_ids[table.idx] = table.is_circular + return circular_contig_ids + + def _get_contig_to_info( + self, reader: H5Reader, contigs_to_keep: set[str] = None + ) -> dict[str, Contig]: + """Load contig metadata for the contigs involved in RGP analysis. + + :param reader: H5Reader opened on the pangenome file. + :param contigs_to_keep: Optional subset of contig names to keep. + :return: Mapping from contig name to Contig metadata records. + """ - rgp_graph.add_nodes_from( - [ident_rgp.ID for ident_rgp in identical_rgp_obj.rgps], - identical_rgp_group=identical_rgp_obj.name, + contig_to_info: dict[str, Contig] = {} + for table in reader.fetch(ContigTable): + if contigs_to_keep is None or table.contig in contigs_to_keep: + contig_to_info[table.contig] = Contig( + organism=table.genome, + is_circular=table.is_circular, + idx=table.idx, + name=table.contig, + ) + + return contig_to_info + + def _fetch_required_table(self, reader: H5Reader, table_type): + """Fetch a required table from the HDF5 file and convert missing-table errors. + + :param reader: H5Reader opened on the pangenome file. + :param table_type: Dataclass schema describing the table to read. + :raises ValueError: If the table does not exist in the file. + :return: Iterator over rows from the requested table. + """ + try: + return reader.fetch(table_type) + except KeyError as e: + raise ValueError( + f"Required table '{table_type.__name__}' is missing from the " + "pangenome H5 file." + ) from e + + def _get_contig_border_genes( + self, reader: H5Reader + ) -> dict[str, ContigBorderPosition]: + """Identify the first and last genes of each contig carrying RGPs. + + :param reader: H5Reader opened on the pangenome file. + :return: Mapping from contig name to its border gene information. + """ + + contig_ids_with_rgp = { + contig_info.idx for contig_info in self._rgp_contig_to_info.values() + } + genedata_to_contig_ids: dict[int, list[int]] = defaultdict(list) + + # Map gene name with genetadata id + # and contig id with genmetadata id in annotations/genes table + + for table in reader.fetch(GenesTable): + if table.contig not in contig_ids_with_rgp: + continue + genedata_to_contig_ids[table.genedata].append(table.contig) + # contig_genedata_id_to_gene_name[(table.contig, table.genedata)] = table.name + + # Create a contig info dictionary to store contig information + contig_to_info: dict[int, ContigBorderPosition] = {} + + for table in reader.fetch(GeneDataTable): + # Problem with RNA genes that are not in GenesTable + # We could filter based on gene_type columns but need to retrieve this column + # and convert it to string when parsing GeneDataTable which takes time + # quick and dirty solution for now: + try: + contig_ids = genedata_to_contig_ids[table.idx] + except KeyError: + continue + for contig_id in contig_ids: + if contig_id not in contig_to_info: + contig_to_info[contig_id] = ContigBorderPosition( + last_gene_position=table.position, + last_gene_idx=table.idx, + first_gene_position=table.position, + first_gene_idx=table.idx, + gene_count=0, + ) + contig_to_info[contig_id].gene_count += 1 + + if table.position > contig_to_info[contig_id].last_gene_position: + contig_to_info[contig_id].last_gene_position = table.position + contig_to_info[contig_id].last_gene_idx = table.idx + + if table.position < contig_to_info[contig_id].first_gene_position: + contig_to_info[contig_id].first_gene_position = table.position + contig_to_info[contig_id].first_gene_idx = table.idx + + assert all( + info.first_gene_position == 0 for info in contig_to_info.values() + ), "Some contigs have no gene at position 0" + + contig_id_to_name = { + contig_info.idx: contig_info.name + for contig_info in self._rgp_contig_to_info.values() + } + + # compute mapping (contig_id, genedata_id) to gene_name info only for border genes + # need to go through GenesTable again to get gene names + contig_genedata_id_to_gene_name = { + (contig_id, info.first_gene_idx): None + for contig_id, info in contig_to_info.items() + } + contig_genedata_id_to_gene_name.update( + { + (contig_id, info.last_gene_idx): None + for contig_id, info in contig_to_info.items() + } ) + for table in reader.fetch(GenesTable): + if (table.contig, table.genedata) in contig_genedata_id_to_gene_name: + contig_genedata_id_to_gene_name[(table.contig, table.genedata)] = ( + table.name + ) - # add edge between identical rgp with metrics at one (perfect score) - edges_to_add = [ - (rgp_a.ID, rgp_b.ID, identical_edge_data) - for rgp_a, rgp_b in combinations(identical_rgp_obj.rgps, 2) - ] - - # replicate all edges that connect identical rgp object to other rgps - for connected_rgp in rgp_graph.neighbors(identical_rgp_obj.ID): - edge_data = rgp_graph[identical_rgp_obj.ID][connected_rgp] - edges_to_add += [ - (identical_rgp.ID, connected_rgp, edge_data) - for identical_rgp in identical_rgp_obj.rgps - ] + contig_name_to_border_genes = { + contig_id_to_name[contig_id]: ContigBorderGenes( + first_gene=contig_genedata_id_to_gene_name[ + contig_id, info.first_gene_idx + ], + last_gene=contig_genedata_id_to_gene_name[ + contig_id, info.last_gene_idx + ], + gene_count=info.gene_count, + ) + for contig_id, info in contig_to_info.items() + } + + return contig_name_to_border_genes - rgp_graph.add_edges_from(edges_to_add) + def _get_rgp_genes(self, reader: H5Reader) -> dict[str, set[str]]: + """Load the gene membership of each RGP from the HDF5 file. - # remove node of the identical rgp object - rgp_graph.remove_node(identical_rgp_obj.ID) + :param reader: H5Reader opened on the pangenome file. + :return: Mapping from each RGP name to the set of genes it contains. + """ + rgp_genes: dict[str, set[str]] = defaultdict(set) - added_identical_rgps += list(identical_rgp_obj.rgps) + for table in reader.fetch(RGPTable): + rgp_genes[table.rgp].add(table.gene) - return added_identical_rgps + return rgp_genes + def _get_metadata_sources( + self, reader: H5Reader, metadata_sources: list[str] = None + ) -> dict[str, list[str]]: + """List metadata sources available for each element type in the pangenome. -def dereplicate_rgp( - rgps: Set[Union[Region, IdenticalRegions]], disable_bar: bool = False -) -> List[Union[Region, IdenticalRegions]]: - """ - Dereplicate RGPs that have the same families. + :param reader: H5Reader opened on the pangenome file. + :param metadata_sources: Optional subset of allowed metadata sources. + :return: Mapping of metatype names to the filtered source list. + """ + status_attrs = reader.handle.root.status._v_attrs + if not (hasattr(status_attrs, "metadata") and status_attrs.metadata): + return {} - Given a list of Region or IdenticalRegions objects representing RGPs, this function groups together - RGPs with the same families into IdenticalRegions objects and returns a list of dereplicated RGPs. + metastatus = reader.handle.root.status.metastatus._v_attrs + metasources = reader.handle.root.status.metasources._v_attrs - :param rgps: A set of Region or IdenticalRegions objects representing the RGPs to be dereplicated. - :param disable_bar: If True, disable the progress bar. + metatype_to_sources: dict[str, list[str]] = {} + for metatype in ("RGPs", "genes", "spots", "families", "modules"): + if not getattr(metastatus, metatype, False): + continue - :return: A list of dereplicated RGPs (Region or IdenticalRegions objects). For RGPs with the same families, - they will be grouped together in IdenticalRegions objects. - """ - logging.getLogger("PPanGGOLiN").info(f"Dereplicating {len(rgps)} RGPs") - families_to_rgps = defaultdict(list) + sources = list(getattr(metasources, metatype, [])) + if metadata_sources is not None: + sources = [source for source in sources if source in metadata_sources] - for rgp in tqdm(rgps, total=len(rgps), unit="RGP", disable=disable_bar): - families_to_rgps[tuple(sorted(f.ID for f in rgp.families))].append(rgp) + if sources: + metatype_to_sources[metatype] = sources + logging.getLogger("PPanGGOLiN").info( + f"Metadata for {metatype} found in pangenome with sources {sources}. " + "They will be included in the RGP graph." + ) + elif metadata_sources is not None: + logging.getLogger("PPanGGOLiN").info( + f"Metadata for {metatype} found in pangenome, but none match the " + f"specified sources {metadata_sources}." + ) - dereplicated_rgps = [] - identical_region_count = 0 - for rgps in families_to_rgps.values(): - if len(rgps) == 1: - dereplicated_rgps.append(rgps[0]) - else: - families = set(rgps[0].families) + return metatype_to_sources - # identical regions object is considered on a contig border if all rgp are contig border - is_contig_border = all(rgp.is_contig_border for rgp in rgps) + def _read_metadata_source( + self, + reader: H5Reader, + metatype: str, + source: str, + value_target: Dict[str, Dict[str, list]] = None, + source_target: Dict[Union[str, int], set] = None, + ) -> None: + """Read one metadata source table and accumulate its values or source flags. + + :param reader: H5Reader opened on the pangenome file. + :param metatype: Metadata element type, such as RGPs, genes, or families. + :param source: Name of the metadata source to read. + :param value_target: Optional mapping of identifiers to metadata value lists. + :param source_target: Optional mapping of identifiers to metadata source sets. + :return: None. The data is stored in the instance dictionaries. + """ + table = reader.handle.get_node(f"/metadata/{metatype}/{source}") + + for row in table.iterrows(): + identifier = row["ID"] + if isinstance(identifier, bytes): + identifier = identifier.decode("utf-8") + + if source_target is not None: + source_target[identifier].add(source) + + if value_target is not None: + for field in table.colnames: + if field == "ID": + continue + value = row[field] + if isinstance(value, bytes): + value = value.decode("utf-8") + value_target[identifier][f"{source}_{field}"].append(str(value)) + + def _load_metadata( + self, reader: H5Reader, metadata_sources: list[str] = None + ) -> None: + """Load metadata from the HDF5 file into the clustering graph metadata stores. + + :param reader: H5Reader opened on the pangenome file. + :param metadata_sources: Optional subset of metadata sources to include. + :return: None. The metadata caches are updated on the instance. + """ + metatype_to_sources = self._get_metadata_sources(reader, metadata_sources) + + for metatype, sources in metatype_to_sources.items(): + for source in sources: + if metatype == "RGPs": + self._read_metadata_source( + reader, metatype, source, value_target=self._rgp_metadata_values + ) + elif metatype == "families": + self._read_metadata_source( + reader, + metatype, + source, + source_target=self._family_metadata_sources, + ) + elif metatype == "genes": + self._read_metadata_source( + reader, + metatype, + source, + source_target=self._gene_metadata_sources, + ) + elif metatype == "spots": + self._read_metadata_source( + reader, + metatype, + source, + source_target=self._spot_metadata_sources, + ) + elif metatype == "modules": + self._read_metadata_source( + reader, + metatype, + source, + source_target=self._module_metadata_sources, + ) + + self._has_metadata = bool(metatype_to_sources) + + def _get_rgp_info( + self, + reader: H5Reader, + rgp_with_genes: dict[str, set[str]], + ) -> list[RGPInfo]: + """Build a compact RGP summary from each gene collection. + + :param reader: H5Reader opened on the pangenome file. + :param rgp_with_genes: Mapping of each RGP name to the genes it contains. + :return: List of RGPInfo objects describing family content and contig status. + """ + + contig_to_border_genes = self._get_contig_border_genes(reader) - # create a new object that will represent the identical rgps - identical_rgp = IdenticalRegions( - name=f"identical_rgps_{identical_region_count}", - identical_rgps=set(rgps), - families=families, + self._rgp_to_nb_genes: dict[str, int] = { + rgp_name: len(genes) for rgp_name, genes in rgp_with_genes.items() + } + + genes_to_keep = {gene for genes in rgp_with_genes.values() for gene in genes} + + gene_to_family: dict[str, str] = {} + + for table in reader.fetch( + GeneFamTable, + override={ + "gene": { + "predicate": lambda x: x.decode("utf-8") in genes_to_keep, + } + }, + ): + gene_to_family[table.gene] = table.family + + # Sort before enumerating: iterating a set of strings depends on + # PYTHONHASHSEED, which is randomized per process by default and would + # make family ID assignment (and therefore node ordering) non-reproducible. + unique_families = sorted(set(gene_to_family.values())) + family_ids = {fam: idx for idx, fam in enumerate(unique_families)} + self._family_id_to_name = {idx: fam for fam, idx in family_ids.items()} + + rgp_infos: list[RGPInfo] = [] + + for rgp_name, genes in rgp_with_genes.items(): + + rgp_families_ids: BitMap = BitMap() + rgp_families: set[str] = set() + contig_name = rgp_name.split("_RGP_")[0] + for gene in genes: + + fam = gene_to_family[gene] + fam_id = family_ids[fam] + rgp_families.add(fam) + rgp_families_ids.add(fam_id) + + contig_border_info = contig_to_border_genes[contig_name] + is_contig_circular = self._rgp_contig_to_info[contig_name].is_circular + + is_contig_border = False + if not is_contig_circular and ( + contig_border_info.first_gene in genes + or contig_border_info.last_gene in genes + ): + is_contig_border = True + + is_whole_contig = False + if len(genes) == contig_border_info.gene_count: + is_whole_contig = True + + info = RGPInfo( + name=rgp_name, + families=rgp_families, + families_ids=rgp_families_ids, is_contig_border=is_contig_border, + is_whole_contig=is_whole_contig, + contig=contig_name, ) - identical_region_count += 1 - dereplicated_rgps.append(identical_rgp) + rgp_infos.append(info) - logging.getLogger("PPanGGOLiN").info(f"{len(dereplicated_rgps)} unique RGPs") - return dereplicated_rgps + return rgp_infos + def _construct_single(self, idx: int, rgp: RGPInfo): + """Create a RegionProxy for a single RGP. -def compute_rgp_metric( - rgp_a: Region, rgp_b: Region, grr_cutoff: float, grr_metric: str -) -> Union[Tuple[int, int, dict], None]: - """ - Compute GRR metric between two RGPs. + :param idx: Stable graph node identifier. + :param rgp: Summary of the RGP to transform into a proxy. + :return: RegionProxy for the single RGP. + """ + return RegionProxy( + ID=idx, + name=rgp.name, + families=rgp.families_ids, + modules=BitMap( + module_id + for fam in rgp.families + for module_id in self._fam_to_modules.get(fam, []) + ), + contig=rgp.contig, + organism=self._rgp_contig_to_info[rgp.contig].organism, + length=self._rgp_to_nb_genes[rgp.name], + is_contig_border=rgp.is_contig_border, + is_whole_contig=rgp.is_whole_contig, + ) - :param rgp_a: A rgp - :param rgp_b: another rgp - :param grr_cutoff: Cutoff filter - :param grr_metric: grr mode between min_grr, max_grr and incomplete_aware_grr + def _construct_single_and_add(self, idx: int, rgp: RGPInfo): + """Create and register a single-region node in the graph. - :returns: Tuple containing the IDs of the two RGPs and the computed metrics as a dictionary - """ + :param idx: Stable graph node identifier. + :param rgp: Summary of the RGP to add. + :return: None. The graph and rgps set are updated. + """ + self.graph.add_node(idx) + self.rgps.add(self._construct_single(idx, rgp)) - edge_metrics = {} + def _construct_multiple(self, idx: int, rgps: list[RGPInfo]): + """Create a merged RegionProxy for identical RGPs. - # RGP at a contig border are seen as incomplete and min GRR is used instead of max GRR - if rgp_a.is_contig_border or rgp_b.is_contig_border: - edge_metrics["incomplete_aware_grr"] = compute_grr( - set(rgp_a.families), set(rgp_b.families), min - ) - else: - edge_metrics["incomplete_aware_grr"] = compute_grr( - set(rgp_a.families), set(rgp_b.families), max + :param idx: Stable graph node identifier for the merged region. + :param rgps: RGP summaries that share the same family content. + :return: RegionProxy representing the identical-group node. + """ + return RegionProxy( + ID=idx, + name=f"identical_rgps_{self.identical_regions}", + families=rgps[0].families_ids, + children=set( + self._construct_single(i, rgp) + for i, rgp in enumerate(rgps, start=idx + 1) + ), + modules=BitMap( + module_id + for fam in rgps[0].families + for module_id in self._fam_to_modules.get(fam, []) + ), + contig=self._contig_to_organism, + # identical regions object is considered on a contig border if all rgp are contig border + is_contig_border=all(rgp.is_contig_border for rgp in rgps), + # identical regions object is considered as whole contig if all rgp are whole contig + is_whole_contig=all(rgp.is_whole_contig for rgp in rgps), ) - # Compute max and min GRR metrics - edge_metrics["max_grr"] = compute_grr(set(rgp_a.families), set(rgp_b.families), max) - edge_metrics["min_grr"] = compute_grr(set(rgp_a.families), set(rgp_b.families), min) + def _construct_multiple_and_add(self, idx: int, rgps: list[RGPInfo]): + """Create and register a merged identical-region node. - # The number of shared families can be useful when visualizing the graph - edge_metrics["shared_family"] = len( - set(rgp_a.families).intersection(set(rgp_b.families)) - ) + :param idx: Stable graph node identifier. + :param rgps: Identical RGP summaries to merge. + :return: None. The graph and rgps collection are updated. + """ + self.rgps.add(self._construct_multiple(idx, rgps)) + self.graph.add_node(idx) + self.identical_regions += 1 - # Only return the metrics if the GRR value is above the cutoff - if edge_metrics[grr_metric] >= grr_cutoff: - return rgp_a.ID, rgp_b.ID, edge_metrics + def _construct_and_add(self, idx: int, rgps: list[RGPInfo]): + """Register a region as either a single node or a merged identical-group node. + :param idx: Stable graph node identifier. + :param rgps: One or more RGP summaries to add. + :return: None. + """ + if len(rgps) == 1: + self._construct_single_and_add(idx, rgps[0]) + else: + self._construct_multiple_and_add(idx, rgps) -def cluster_rgp_on_grr(graph: nx.Graph, clustering_attribute: str = "grr"): - """ - Cluster rgp based on grr using louvain communities clustering. + def _grr(self, b1: BitMap, b2: BitMap, mode: Callable) -> float: + """Compute the gene repertoire relatedness between two family sets. - :param graph: NetworkX graph object representing the RGPs and their relationship - :param clustering_attribute: Attribute of the graph to use for clustering (default is "grr") - """ + :param b1: BitMap of families in the first region. + :param b2: BitMap of families in the second region. + :param mode: Callable used to choose the denominator (min or max). + :return: Jaccard-style similarity value between the two family sets. + """ + return len(b1 & b2) / mode(len(b1), len(b2)) + + def _rgp_metric( + self, r1: RegionProxy, r2: RegionProxy, grr_cutoff: float, metric: RGPMetricType + ) -> RGPMetric: + """Compute the similarity metric used to connect two RGPs in the graph. + + :param r1: First region proxy. + :param r2: Second region proxy. + :param grr_cutoff: Minimum required metric value for an edge to be retained. + :param metric: Name of the metric to evaluate. + :return: RGPMetric if the similarity exceeds the cutoff, otherwise None. + """ + if r1.is_contig_border or r2.is_contig_border: + agrr = self._grr(r1.families, r2.families, min) + max_grr = self._grr(r1.families, r2.families, max) + min_grr = agrr + else: + agrr = self._grr(r1.families, r2.families, max) + min_grr = self._grr(r1.families, r2.families, min) + max_grr = agrr - partitions = nx.algorithms.community.louvain_communities( - graph, weight=clustering_attribute - ) + m = RGPMetric(max_grr, min_grr, agrr, len(r1.families & r2.families)) + return m if getattr(m, metric) >= grr_cutoff else None - # Add partition index in node attributes - for i, cluster_nodes in enumerate(partitions): - nx.set_node_attributes( - graph, - {node: f"cluster_{i}" for node in cluster_nodes}, - name=f"{clustering_attribute}_cluster", - ) + def _construct_regions( + self, + with_metadata: bool = False, + metadata_sources: list[str] = None, + ignore_incomplete_rgp: bool = False, + ): + """Load all RGPs from disk and assemble graph nodes before clustering. - logging.getLogger("PPanGGOLiN").info( - f"Graph has {len(partitions)} clusters using {clustering_attribute}" - ) + :param with_metadata: Whether metadata should be attached to the graph. + :param metadata_sources: Optional list of metadata sources to include. + :param ignore_incomplete_rgp: Whether edge-border RGPs should be filtered out. + :return: None. The graph content and RGP cache are populated. + """ + logging.getLogger("PPanGGOLiN").info("Loading RGPs from pangenome H5 file") + with H5Reader(self.h5) as reader: -def get_spot_id(rgp: Region, rgp_to_spot: Dict[Region, int]) -> str: - """ - Return Spot ID associated to an RGP. - It adds the prefix "spot" to the spot ID. When no spot is associated with the RGP, - then the string "No spot" is return + rgp_to_genes = self._get_rgp_genes(reader) + if with_metadata: + self._rgp_to_genes = rgp_to_genes - :param rgp: RGP id - :param rgp_to_spot: A dictionary mapping an RGP to its spot. + contigs_with_rgp = {rgp_name.split("_RGP_")[0] for rgp_name in rgp_to_genes} - :return: Spot ID of the given RGP with the prefix spot or "No spot". - """ - if rgp in rgp_to_spot: - return f"spot_{rgp_to_spot[rgp]}" - else: - return "No spot" - - -def write_rgp_cluster_table( - outfile: str, - grr_graph: nx.Graph, - rgps_in_graph: List[Union[Region, IdenticalRegions]], - grr_metric: str, - rgp_to_spot: Dict[Region, int], -) -> None: - """ - Writes RGP cluster info to a TSV file using pandas. - - :param outfile: Name of the tsv file - :param grr_graph: The GRR graph. - :param rgps_in_graph: A dictionary mapping an RGP to a set of identical RGPs. - :param grr_metric: The GRR metric used for clustering. - :param rgp_to_spot: A dictionary mapping an RGP to its spot. - :return: None - """ + self._rgp_contig_to_info = self._get_contig_to_info( + reader, contigs_with_rgp + ) - all_rgps_infos = [] - for rgp_in_graph in rgps_in_graph: - cluster = grr_graph.nodes[rgp_in_graph.ID][f"{grr_metric}_cluster"] + rgp_infos = self._get_rgp_info(reader, rgp_to_genes) - identical_rgps = ( - [rgp_in_graph] if isinstance(rgp_in_graph, Region) else rgp_in_graph.rgps - ) + if ignore_incomplete_rgp: + rgp_infos = [ + rgp_info for rgp_info in rgp_infos if not rgp_info.is_contig_border + ] + logging.getLogger("PPanGGOLiN").info( + f"{len(rgp_infos)} RGPs loaded from pangenome after filtering out incomplete RGPs" + ) - all_rgps_infos += [ - {"RGPs": r.name, "cluster": cluster, "spot_id": get_spot_id(r, rgp_to_spot)} - for r in identical_rgps - ] - - df = pd.DataFrame(all_rgps_infos) - df.to_csv(outfile, sep="\t", index=False) - - -def cluster_rgp( - pangenome, - grr_cutoff: float, - output: str, - basename: str, - ignore_incomplete_rgp: bool, - unmerge_identical_rgps: bool, - grr_metric: str, - disable_bar: bool, - graph_formats: Set[str], - add_metadata: bool = False, - metadata_sep: str = "|", - metadata_sources: List[str] = None, -): - """ - Main function to cluster regions of genomic plasticity based on their GRR - - :param pangenome: pangenome object - :param grr_cutoff: GRR cutoff value for clustering - :param output: Directory where the output files will be saved - :param basename: Basename for the output files - :param ignore_incomplete_rgp: Whether to ignore incomplete RGPs located at a contig border - :param unmerge_identical_rgps: Whether to unmerge identical RGPs into separate nodes in the graph - :param grr_metric: GRR metric to use for clustering - :param disable_bar: Whether to disable the progress bar - :param graph_formats: Set of graph file formats to save the output - :param add_metadata: Add metadata to cluster files - :param metadata_sep: The separator used to join multiple metadata values - :param metadata_sources: Sources of the metadata to use and write in the outputs. None means all sources are used. - """ + self._rgp_to_spot = self._get_rgp_spot(reader) + self._fam_to_modules = self._get_fam_to_modules(reader) - metatypes = set() - need_metadata = False - if add_metadata: - for element in ["RGPs", "genes", "spots", "families", "modules"]: - if pangenome.status["metadata"][element] == "inFile": + if with_metadata: + self._load_metadata(reader, metadata_sources) - sources_to_use = set(pangenome.status["metasources"][element]) + fams_to_rgps: defaultdict[tuple[int], list[RGPInfo]] = defaultdict(list) - if metadata_sources is not None: - if ( - len( - set(pangenome.status["metasources"][element]) - & set(metadata_sources) - ) - == 0 - ): - logging.getLogger("PPanGGOLiN").info( - f"Metadata for {element} found in pangenome, but none match the specified sources {metadata_sources}. " - f"Current source for {element}: {sources_to_use}." - ) + for info in rgp_infos: + fams_key = tuple(sorted(fam_id for fam_id in info.families_ids)) + fams_to_rgps[fams_key].append(info) + + idx = 0 + for _fams, rgps in sorted(fams_to_rgps.items(), key=lambda x: x[0]): + self._construct_and_add(idx, rgps) + idx += len(rgps) + 1 if len(rgps) > 1 else 1 + + logging.getLogger("PPanGGOLiN").info( + f"{len(rgp_infos)} RGPs loaded from pangenome ({len(self.rgps)} unique RGPs after dereplication)" + ) + if len(rgp_infos) == 0: + logging.getLogger("PPanGGOLiN").warning( + "Pangenome contains no RGPs. Output files will be empty." + ) + + def _compute_all_metrics( + self, grr_cutoff: float, metric: RGPMetricType, disable_bar: bool = False + ): + """Compute RGP-to-RGP similarity metrics and retain edges above the threshold. + + :param grr_cutoff: Minimum similarity required to keep an edge. + :param metric: Metric name used to decide edge retention. + :param disable_bar: Whether to silence the progress bar. + :return: None. The graph edges are updated in place. + """ + logging.getLogger("PPanGGOLiN").info("Computing RGP metrics") + + family_to_rgps = defaultdict(list) + for rgp in self.rgps: + for family in rgp.families: + family_to_rgps[family].append(rgp) + + nb_pairs = 0 + selected_count = 0 + + with tqdm( + total=len(family_to_rgps), + desc="Computing metrics", + disable=disable_bar, + bar_format="{desc}: {percentage:3.0f}%|{bar}", + ) as pbar: + for family in sorted(family_to_rgps): + for r1, r2 in combinations(family_to_rgps[family], 2): + shared_families = r1.families & r2.families + if family != next(iter(shared_families)): continue - else: - sources_to_use = set( - pangenome.status["metasources"][element] - ) & set(metadata_sources) - need_metadata = True - metatypes.add(element) - logging.getLogger("PPanGGOLiN").info( - f"Metadata for {element} found in pangenome with sources {sources_to_use}. They will be included in the RGP graph." - ) + nb_pairs += 1 + if m := self._rgp_metric(r1, r2, grr_cutoff, metric): + selected_count += 1 + self.graph.add_edge(r1.ID, r2.ID, **m.__dict__) + pbar.update(1) - # check statuses and load info - check_pangenome_info( - pangenome, - need_families=True, - need_annotations=True, - disable_bar=disable_bar, - need_rgp=True, - need_spots=True, - need_modules=True, - need_metadata=need_metadata, - sources=metadata_sources, - metatypes=metatypes, - ) + logging.getLogger("PPanGGOLiN").info( + f"RGP metrics computed for {nb_pairs:,} pairs of RGPs " + f"({selected_count:,} selected after GRR cutoff)" + ) + + def _louvain_clustering(self, metric: RGPMetricType): + """Cluster the RGP graph using the Louvain method. - if pangenome.regions == 0: - raise Exception( - "The pangenome has no RGPs. The clustering of RGP is then not possible." + :param metric: Similarity metric to use as the graph edge weight. + :return: None. Cluster labels are attached to graph nodes. + """ + logging.getLogger("PPanGGOLiN").info( + f"Clustering RGPs using Louvain communities on '{metric}' metric" ) - # add all rgp as node - if ignore_incomplete_rgp: - valid_rgps = [rgp for rgp in pangenome.regions if not rgp.is_contig_border] + partitions = nx.algorithms.community.louvain_communities( + self.graph, weight=metric, seed=42 + ) + ordered_partitions = sorted( + partitions, + key=lambda nodes: (min(nodes), len(nodes)), + ) - ignored_rgp_count = pangenome.number_of_rgp - len(valid_rgps) - total_rgp_count = pangenome.number_of_rgp + for i, nodes in enumerate(ordered_partitions): + nx.set_node_attributes( + self.graph, + {node: f"cluster_{i}" for node in nodes}, + name=f"{metric}_cluster", + ) logging.getLogger("PPanGGOLiN").info( - f"Ignoring {ignored_rgp_count}/{total_rgp_count} ({100 * ignored_rgp_count / total_rgp_count:.2f}%) " - "RGPs that are located at a contig border and are likely incomplete." + f"Graph has {len(ordered_partitions)} RGP clusters using '{metric}'" ) - if len(valid_rgps) == 0: - raise Exception( - "The pangenome has no complete RGPs. The clustering of RGP is then not possible." + def _add_edges_to_identical_rgps(self): + """Expand merged identical-region nodes into their child RGP nodes and edges. + + :return: None. The graph is updated to include both children and the original group. + """ + logging.getLogger("PPanGGOLiN").info("Unmerging identical RGPs in the graph") + + unmerged = 0 + edge_data = { + "max_grr": 1.0, + "min_grr": 1.0, + "grr": 1.0, + "identical_famillies": True, + } + + for rgp in self.rgps: + if not rgp.is_identical_region: + continue + + unmerged += 1 + self.graph.add_nodes_from( + (child.ID for child in rgp.children), + identical_rgp_group=rgp.name, ) - else: - valid_rgps = set(pangenome.regions) - dereplicated_rgps = dereplicate_rgp(valid_rgps, disable_bar=disable_bar) + edges = [ + (r1.ID, r2.ID, edge_data) for r1, r2 in combinations(rgp.children, 2) + ] - grr_graph = nx.Graph() - grr_graph.add_nodes_from(rgp.ID for rgp in dereplicated_rgps) + for connected in self.graph.neighbors(rgp.ID): + data = self.graph[rgp.ID][connected] + edges += [(child.ID, connected, data) for child in rgp.children] - # Get all pairs of RGP that share at least one family + self.graph.add_edges_from(edges) + self.graph.remove_node(rgp.ID) - family2rgp = defaultdict(set) - for rgp in dereplicated_rgps: - for fam in rgp.families: - family2rgp[fam].add(rgp) + logging.getLogger("PPanGGOLiN").info(f"Unmerged {unmerged} identical RGPs") - rgp_pairs = set() - for rgps in family2rgp.values(): - rgp_pairs |= {tuple(sorted(rgp_pair)) for rgp_pair in combinations(rgps, 2)} + def _spot_id(self, rgp: RegionProxy) -> str: + """Return the human-readable spot identifier for an RGP. - pairs_count = len(rgp_pairs) + :param rgp: Region proxy to inspect. + :return: Spot label in the form 'spot_X' or 'No spot'. + """ + if rgp.name in self._rgp_to_spot: + return f"spot_{self._rgp_to_spot[rgp.name]}" + else: + return "No spot" - logging.getLogger("PPanGGOLiN").info( - f"Computing GRR metric for {pairs_count:,} pairs of RGP." - ) + def _add_metadata_info( + self, + info: dict, + rgp_names: List[str], + families_ids: BitMap, + module_ids: Set[int], + ) -> None: + """Append metadata-derived attributes to an RGP node description. + + :param info: Dictionary storing node attributes to enrich. + :param rgp_names: Names of the RGPs to inspect for metadata. + :param families_ids: Family identifiers contributing to the metadata summary. + :param module_ids: Module identifiers contributing to the metadata summary. + :return: None. The info dictionary is modified in place. + """ + if not self._has_metadata: + return + + element_to_sources = { + "family": ( + set().union( + *( + self._family_metadata_sources.get( + self._family_id_to_name[fid], set() + ) + for fid in families_ids + ) + ) + if families_ids + else set() + ), + "module": ( + set().union( + *( + self._module_metadata_sources.get(module_id, set()) + for module_id in module_ids + ) + ) + if module_ids + else set() + ), + "gene": set(), + "spot": set(), + } - pairs_of_rgps_metrics = [] + for rgp_name in rgp_names: + for gene_name in self._rgp_to_genes.get(rgp_name, ()): + element_to_sources["gene"] |= self._gene_metadata_sources.get( + gene_name, set() + ) - for rgp_a, rgp_b in rgp_pairs: + spot_id = self._rgp_to_spot.get(rgp_name) + if spot_id is not None: + element_to_sources["spot"] |= self._spot_metadata_sources.get( + spot_id, set() + ) - pair_metrics = compute_rgp_metric(rgp_a, rgp_b, grr_cutoff, grr_metric) + for element, sources in element_to_sources.items(): + for source in sources: + info[f"has_{element}_with_{source}"] = True - if pair_metrics: - pairs_of_rgps_metrics.append(pair_metrics) + combined_rgp_metadata: Dict[str, list] = defaultdict(list) + for rgp_name in rgp_names: + for key, values in self._rgp_metadata_values.get(rgp_name, {}).items(): + combined_rgp_metadata[key].extend(values) - grr_graph.add_edges_from(pairs_of_rgps_metrics) + for key, values in combined_rgp_metadata.items(): + info[key] = json.dumps(values) - identical_rgps_objects = [ - rgp for rgp in dereplicated_rgps if isinstance(rgp, IdenticalRegions) - ] - rgp_objects_in_graph = [rgp for rgp in dereplicated_rgps if isinstance(rgp, Region)] + def _make_info_identical_rgp(self, rgp: RegionProxy) -> dict: + """Build the attribute dictionary for a merged identical-RGP group node. + + :param rgp: Region proxy representing identical child RGPs. + :return: Dictionary of node attributes for output tables and graph exports. + """ + + spots = {self._spot_id(child) for child in rgp.children} + + info = { + "identical_rgp_group": True, + "name": rgp.name, + "families_count": len(rgp.families), + "identical_rgp_count": len(rgp.children), + "identical_rgp_names": ";".join( + sorted(child.name for child in rgp.children) + ), + "identical_rgp_genomes": ";".join( + ";".join(sorted({child.organism for child in rgp.children})) + ), + "identical_rgp_contig_border_count": len( + [True for child in rgp.children if child.is_contig_border] + ), + "identical_rgp_whole_contig_count": len( + [True for child in rgp.children if child.is_whole_contig] + ), + "identical_rgp_spots": ";".join(sorted(spots)), + "spot_id": (spots.pop() if len(spots) == 1 else "Multiple spots"), + "modules": ";".join(sorted(f"module_{module}" for module in rgp.modules)), + } - if unmerge_identical_rgps: - rgp_objects_in_graph += add_edges_to_identical_rgps( - grr_graph, identical_rgps_objects + self._add_metadata_info( + info, + sorted([child.name for child in rgp.children]), + rgp.families, + rgp.modules, ) - # cluster rgp based on grr value - logging.getLogger("PPanGGOLiN").info( - f"Louvain_communities clustering of RGP based on {grr_metric} on {grr_graph}." - ) + return info + + def _make_info_from_rgp(self, rgp: RegionProxy) -> dict: + """Build the attribute dictionary for a single RGP node. + + :param rgp: Region proxy representing a single RGP. + :return: Dictionary of node attributes for output tables and graph exports. + """ + + info = { + "contig": rgp.contig, + "genome": rgp.organism, + "name": rgp.name, + "genes_count": rgp.length, + "is_contig_border": rgp.is_contig_border, + "is_whole_contig": rgp.is_whole_contig, + "spot_id": self._spot_id(rgp), + "modules": ";".join(f"module_{module}" for module in rgp.modules), + "families_count": rgp.nb_families, + } + + self._add_metadata_info(info, [rgp.name], rgp.families, rgp.modules) + + return info + + def _add_info_to_rgps(self): + """Attach metadata and summary attributes to each graph node. + + :return: None. Each node in the graph is updated with annotation fields. + """ + + logging.getLogger("PPanGGOLiN").info("Adding info to RGPs in graph") + + annotated = 0 + for rgp in self.rgps: + + if rgp.ID in self.graph: + if rgp.is_identical_region: + self.graph.nodes[rgp.ID].update(self._make_info_identical_rgp(rgp)) + + else: + self.graph.nodes[rgp.ID].update(self._make_info_from_rgp(rgp)) + + annotated += 1 - cluster_rgp_on_grr(grr_graph, grr_metric) + if rgp.children: # in case identical rgp are unmerged + for child in rgp.children: + if child.ID in self.graph: + self.graph.nodes[child.ID].update( + self._make_info_from_rgp(child) + ) + annotated += 1 + + logging.getLogger("PPanGGOLiN").info(f"Added info to {annotated} RGPs") + + def _write_graphs(self, output: Path, basename: str, graph_formats: list[str]): + """Write the computed graph to the selected output formats. + + :param output: Output directory for the graph files. + :param basename: Basename used for the exported files. + :param graph_formats: Graph output formats to generate. + :return: None. Files are written on disk. + """ + if "gexf" in graph_formats: + graph_filename = output / f"{basename}.gexf" + logging.getLogger("PPanGGOLiN").info( + f"Writing RGP graph in GEXF format to {graph_filename}" + ) + nx.write_gexf(self.graph, graph_filename) + + if "graphml" in graph_formats: + graph_filename = output / f"{basename}.graphml" + logging.getLogger("PPanGGOLiN").info( + f"Writing RGP graph in GraphML format to {graph_filename}" + ) + nx.write_graphml(self.graph, graph_filename) - rgp_to_spot = { - region: int(spot.ID) for spot in pangenome.spots for region in spot.regions - } + def _write_cluster_table(self, output: Path, basename: str, metric: str): + """Write cluster assignments to a TSV table. - if not unmerge_identical_rgps: + :param output: Output directory for the TSV file. + :param basename: Basename used for the output file. + :param metric: Metric name used in the cluster attribute labels. + :return: None. The cluster table is written to disk. + """ + outfile = output / f"{basename}.tsv" + rows = [] + + for node, attrs in self.graph.nodes(data=True): + cluster = attrs.get(f"{metric}_cluster") + if cluster is None: + raise ValueError( + f"Node {node} does not have a '{metric}_cluster' attribute" + ) + + name = attrs["name"] + + if name.startswith("identical_rgps_"): + identical_rgp_names = attrs["identical_rgp_names"] + + for child_name in identical_rgp_names.split(";"): + rows.append( + { + "RGPs": child_name, + "cluster": cluster, + "spot_id": self._rgp_to_spot.get(child_name, "No spot"), + } + ) + else: + + rows.append( + { + "RGPs": attrs.get("name", str(node)), + "cluster": cluster, + "spot_id": attrs.get("spot_id", "No spot"), + } + ) + + rows.sort(key=lambda row: (row["cluster"], row["RGPs"])) + pd.DataFrame(rows, columns=["RGPs", "cluster", "spot_id"]).to_csv( + outfile, + sep="\t", + index=False, + ) logging.getLogger("PPanGGOLiN").info( - "Add info on identical RGPs merged in the graph" + f"Writing RGP clusters in TSV format to {outfile}" ) - add_info_to_identical_rgps(grr_graph, identical_rgps_objects, rgp_to_spot) - rgps_in_graph = ( - rgp_objects_in_graph if unmerge_identical_rgps else dereplicated_rgps - ) + def _write_outputs( + self, output: Path, basename: str, graph_formats: list[str], metric: str + ): + """Write all clustering outputs for the current analysis. - # add some attribute to the graph nodes. - logging.getLogger("PPanGGOLiN").info("Add RGP information to the graph") - add_info_to_rgp_nodes(grr_graph, rgp_objects_in_graph, rgp_to_spot) + :param output: Output directory for the exported files. + :param basename: Basename used for all generated files. + :param graph_formats: Output graph formats to generate. + :param metric: Metric name used in cluster labels. + :return: None. Output tables and graphs are written on disk. + """ + self._write_graphs(output, basename, graph_formats) + self._write_cluster_table(output, basename, metric) - if need_metadata: - add_rgp_metadata_to_graph(grr_graph, rgps_in_graph) + def run(self, options: RGPClusteringOptions): + """Run the full RGP clustering workflow from HDF5 loading to output writing. - if "gexf" in graph_formats: - # writing graph in gexf format - graph_file_name = os.path.join(output, f"{basename}.gexf") - logging.getLogger("PPanGGOLiN").info( - f"Writing graph in gexf format in {graph_file_name}." + :param options: Configuration parameters for the clustering process. + :return: None. The graph and output files are generated in place. + """ + + self._construct_regions( + with_metadata=options.with_metadata, + metadata_sources=options.metadata_sources or None, + ignore_incomplete_rgp=options.ignore_incomplete_rgp, ) - nx.readwrite.gexf.write_gexf(grr_graph, graph_file_name) - if "graphml" in graph_formats: - graph_file_name = os.path.join(output, f"{basename}.graphml") - logging.getLogger("PPanGGOLiN").info( - f"Writing graph in graphml format in {graph_file_name}." + self._compute_all_metrics( + options.grr_cutoff, options.metric, disable_bar=options.disable_prog_bar ) - nx.readwrite.graphml.write_graphml(grr_graph, graph_file_name) - outfile = os.path.join(output, f"{basename}.tsv") - logging.getLogger("PPanGGOLiN").info( - f"Writing rgp clusters in tsv format in {outfile}" - ) + if options.unmerge_identical_rgps: + self._add_edges_to_identical_rgps() + + self._louvain_clustering(options.metric) + + self._add_info_to_rgps() - write_rgp_cluster_table(outfile, grr_graph, rgps_in_graph, grr_metric, rgp_to_spot) + self._write_outputs( + options.output, options.basename, options.graph_formats, options.metric + ) + + @property + def rgp_count(self) -> int: + """Return the number of unique RGP proxies held in memory. + + :return: Count of loaded RGP entries. + """ + return len(self.rgps) def launch(args: argparse.Namespace): - """ - Command launcher + """Command launcher for the RGP clustering workflow. - :param args: All arguments provided by user + :param args: All arguments provided by the user on the command line. + :return: None. The clustering workflow is executed in place. """ - pangenome = Pangenome() mk_outdir(args.output, args.force) - pangenome.add_file(args.pangenome) - - cluster_rgp( - pangenome, - grr_cutoff=args.grr_cutoff, - output=args.output, - basename=args.basename, - ignore_incomplete_rgp=args.ignore_incomplete_rgp, - unmerge_identical_rgps=args.no_identical_rgp_merging, - grr_metric=args.grr_metric, - disable_bar=args.disable_prog_bar, - graph_formats=args.graph_formats, - add_metadata=args.add_metadata, - metadata_sep=args.metadata_sep, - metadata_sources=args.metadata_sources, + if args.metadata_sep is not None: + logging.getLogger("PPanGGOLiN").warning( + "--metadata_sep is obsolete and ignored; metadata values are JSON-encoded in graphs." + ) + + clustering = RGPClustering(args.pangenome) + clustering.run( + RGPClusteringOptions( + unmerge_identical_rgps=args.no_identical_rgp_merging, + grr_cutoff=args.grr_cutoff, + metric=args.grr_metric, + output=args.output, + basename=args.basename, + graph_formats=args.graph_formats, + with_metadata=args.add_metadata, + metadata_sources=args.metadata_sources, + ignore_incomplete_rgp=args.ignore_incomplete_rgp, + disable_prog_bar=args.disable_prog_bar, + ) ) def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser: - """ - Subparser to launch PPanGGOLiN in Command line + """Create the command-line subparser for the RGP clustering command. - :param sub_parser : Sub_parser for cluster_rgp command - - :return : Parser arguments for cluster_rgp command + :param sub_parser: Argument parser collection that receives the new subcommand. + :return: The configured argument parser for the RGP clustering command. """ parser = sub_parser.add_parser( "rgp_cluster", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Cluster RGPs based on their gene families." + parser.category = "Regions of Genomic Plasticity" parser_cluster_rgp(parser) return parser def parser_cluster_rgp(parser: argparse.ArgumentParser): - """ - Parser for specific argument of rgp command + """Define all CLI arguments for the RGP clustering command. - :param parser: Parser for cluster_rgp argument + :param parser: Argument parser instance to enrich with clustering options. + :return: None. The passed parser is updated in place. """ required = parser.add_argument_group( title="Required arguments", description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=True, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=True, + type=Path, + help="Path to the pangenome .h5 file.", ) optional = parser.add_argument_group(title="Optional arguments") @@ -868,7 +1456,7 @@ def parser_cluster_rgp(parser: argparse.ArgumentParser): "--basename", required=False, default="rgp_cluster", - help="basename for the output file", + help="Basename for the output file.", ) optional.add_argument( @@ -877,7 +1465,7 @@ def parser_cluster_rgp(parser: argparse.ArgumentParser): required=False, type=Path, default="rgp_clustering", - help="Output directory", + help="Output directory.", ) optional.add_argument( @@ -886,7 +1474,7 @@ def parser_cluster_rgp(parser: argparse.ArgumentParser): type=str, choices=["gexf", "graphml"], nargs="+", - default=["gexf", "graphml"], + default=["graphml"], help="Format of the output graph.", ) @@ -909,7 +1497,6 @@ def parser_cluster_rgp(parser: argparse.ArgumentParser): optional.add_argument( "--metadata_sep", required=False, - default="|", - help="The separator used to join multiple metadata values for elements with multiple metadata" - " values from the same source. This character should not appear in metadata values.", + default=None, + help=argparse.SUPPRESS, ) diff --git a/ppanggolin/RGP/spot.py b/ppanggolin/RGP/spot.py index df76e480..2da3746e 100644 --- a/ppanggolin/RGP/spot.py +++ b/ppanggolin/RGP/spot.py @@ -335,6 +335,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "spot", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Predicts spots in your pangenome." + parser.category = "Regions of Genomic Plasticity" parser_spot(parser) return parser @@ -350,7 +352,11 @@ def parser_spot(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) optional = parser.add_argument_group(title="Optional arguments") optional.add_argument( @@ -362,7 +368,7 @@ def parser_spot(parser: argparse.ArgumentParser): f"ppanggolin_output{time.strftime('DATE%Y-%m-%d_HOUR%H.%M.%S', time.localtime())}" f"_PID{str(os.getpid())}" ), - help="Output directory", + help="Output directory.", ) optional.add_argument( "--spot_graph", diff --git a/ppanggolin/__init__.py b/ppanggolin/__init__.py index 27110710..6a2eb65e 100755 --- a/ppanggolin/__init__.py +++ b/ppanggolin/__init__.py @@ -14,6 +14,7 @@ import ppanggolin.workflow import ppanggolin.projection import ppanggolin.meta +import ppanggolin.info SUBCOMMAND_TO_SUBPARSER = { "annotate": ppanggolin.annotate.subparser, @@ -33,6 +34,7 @@ "msa": ppanggolin.formats.writeMSA.subparser, "metrics": ppanggolin.metrics.metrics.subparser, "align": ppanggolin.align.subparser, + "info": ppanggolin.info.subparser, "rgp": ppanggolin.RGP.genomicIsland.subparser, "spot": ppanggolin.RGP.spot.subparser, "module": ppanggolin.mod.subparser, @@ -42,6 +44,13 @@ "metadata": ppanggolin.meta.subparser, } +# The utility command lives in its own module, so we import it only after the +# package is initialized and register it here to keep the CLI subcommand registry +# in one place. This is the canonical list consumed by build_parser(). +import ppanggolin.utility + +SUBCOMMAND_TO_SUBPARSER["utils"] = ppanggolin.utility.utils.subparser + version = distribution("ppanggolin").version __version__ = version diff --git a/ppanggolin/align/alignOnPang.py b/ppanggolin/align/alignOnPang.py index 949f336b..505db94f 100644 --- a/ppanggolin/align/alignOnPang.py +++ b/ppanggolin/align/alignOnPang.py @@ -859,6 +859,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "align", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Aligns a genome or a set of proteins to the pangenome gene families and predicts information from it." + parser.category = "Analysis using reference pangenomes" parser_align(parser) return parser @@ -878,18 +880,22 @@ def parser_align(parser: argparse.ArgumentParser): "--sequences", required=False, type=Path, - help="sequences (nucleotides or amino acids) to align on the pangenome gene families", + help="Sequences (nucleotides or amino acids) to align against the pangenome gene families.", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) required.add_argument( "-o", "--output", required=True, type=Path, - help="Output directory where the file(s) will be written", + help="Output directory where the file(s) will be written.", ) optional = parser.add_argument_group(title="Optional arguments") @@ -897,22 +903,22 @@ def parser_align(parser: argparse.ArgumentParser): "--no_defrag", required=False, action="store_true", - help="DO NOT Realign gene families to link fragments with" - "their non-fragmented gene family. (default: False)", + help="Do not realign gene families to link fragments with their non-fragmented gene family." + " (Default: False)", ) optional.add_argument( "--identity", required=False, type=float, default=0.5, - help="min identity percentage threshold", + help="Minimum identity percentage threshold.", ) optional.add_argument( "--coverage", required=False, type=float, default=0.8, - help="min coverage percentage threshold", + help="Minimum coverage percentage threshold.", ) optional.add_argument( "--fast", @@ -934,15 +940,14 @@ def parser_align(parser: argparse.ArgumentParser): "--getinfo", required=False, action="store_true", - help="Use this option to extract info related to the best hit of each query, " - "such as the RGP it is in, or the spots.", + help="Use this option to extract information related to the best hit of each query, " + "such as the RGP it is in or the spots it belongs to.", ) optional.add_argument( "--draw_related", required=False, action="store_true", - help="Draw figures and provide graphs in a gexf format of the eventual spots" - " associated to the input sequences", + help="Draw figures and provide graphs in GEXF format for the spots associated with the input sequences.", ) # but does not use the option optional.add_argument( @@ -950,7 +955,7 @@ def parser_align(parser: argparse.ArgumentParser): required=False, action="store_true", help="In the context of provided annotation, use this option to read pseudogenes. " - "(Default behavior is to ignore them)", + "(Default behavior is to ignore them).", ) optional.add_argument( "-c", @@ -958,21 +963,21 @@ def parser_align(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) optional.add_argument( "--tmpdir", required=False, type=Path, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) optional.add_argument( "--keep_tmp", required=False, default=False, action="store_true", - help="Keeping temporary files (useful for debugging).", + help="Keep temporary files (useful for debugging).", ) diff --git a/ppanggolin/annotate/annotate.py b/ppanggolin/annotate/annotate.py index 111cd17b..1e02f13d 100644 --- a/ppanggolin/annotate/annotate.py +++ b/ppanggolin/annotate/annotate.py @@ -5,20 +5,19 @@ import logging from concurrent.futures import ProcessPoolExecutor from itertools import chain - from multiprocessing import get_context import os from pathlib import Path import tempfile import time from typing import List, Set, Tuple, Iterable, Dict, Generator, Union -import re from collections import defaultdict, Counter import warnings # installed libraries from tqdm import tqdm from tables.path import check_name_validity, NaturalNameWarning +import gb_io # local libraries from ppanggolin.annotate.synta import ( @@ -37,6 +36,7 @@ check_input_files, has_non_ascii, replace_non_ascii, + parse_input_paths_file, ) from ppanggolin.formats import write_pangenome from ppanggolin.metadata import Metadata @@ -156,286 +156,65 @@ def create_gene( return new_gene -def extract_positions(string: str) -> Tuple[List[Tuple[int, int]], bool, bool, bool]: +def extract_positions( + location: gb_io.Complement | gb_io.Join | gb_io.Range | gb_io.Order, +) -> tuple[list[tuple[int, int]], bool, bool, bool]: """ - Extracts start and stop positions from a string and determines whether it is complement and pseudogene. - - Example of strings that the function is able to process: + Extracts start and stop positions from a location Class from gb.io (Complement, Join, Range) and returns a tuple containing the coordinates, whether it is a complement, and whether it has partial start or end. - "join(190..7695,7695..12071)", - "complement(join(4359800..4360707,4360707..4360962))", - "join(6835405..6835731,1..1218)", - "join(1375484..1375555,1375557..1376579)", - "complement(6815492..6816265)", - "6811501..6812109", - "complement(6792573..>6795461)", - "join(1038313,1..1016)" - - :param string: The input string containing position information. + :param location: The input location object containing position information. :return: A tuple containing a list of tuples representing start and stop positions, a boolean indicating whether it is complement, a boolean indicating whether it is a partial gene at start position and a boolean indicating whether it is a partial gene at end position. - :raises ValueError: If the string is not formatted as expected or if positions cannot be parsed as integers. + :raises ValueError: If the location is not formatted as expected or if positions cannot be parsed as integers. """ - complement = False - coordinates = [] + is_complement = False has_partial_start = False has_partial_end = False + coordinates = [] - # Check if 'complement' exists in the string - if "complement" in string: - complement = True - - if "(" in string: - # Extract positions found inside the parenthesis - inner_parentheses_regex = r"\(([^()]+)\)" - inner_matches = re.findall(inner_parentheses_regex, string) + if isinstance(location, gb_io.Complement): + is_complement = True + location = location.location - try: - positions = inner_matches[-1] - except IndexError: - raise ValueError(f"Gene position {string} is not formatted as expected.") + if isinstance(location, (gb_io.Join, gb_io.Order)): # process them the same for now + parts = location.locations + elif isinstance(location, gb_io.Range): + parts = [location] else: - positions = string.rstrip() - - # Check if '>' or '<' exists in the positions to identify partial genes - - if ">" in positions or "<" in positions: - if "<" in positions.split(",")[0]: - has_partial_start = True - - if ">" in positions.split(",")[-1]: - has_partial_end = True - - inner_positions = ",".join(positions.split(",")[1:-1]) - - if ( - ">" in inner_positions - or "<" in inner_positions - or (not has_partial_end and not has_partial_start) - ): - raise ValueError( - f"Error parsing positions '{positions}' extracted from GBFF string '{string}'. " - f"Chevrons are found in the inner position. This case is unexpected and not handle." - ) - - for position in positions.split(","): - try: - start, stop = position.replace(">", "").replace("<", "").split("..") - except ValueError: - # in some case there is only one position meaning that the gene is long of only one nt in this piece. - # for instance : join(1038313,1..1016) - start = position.replace(">", "").replace("<", "") - stop = start - try: - start, stop = int(start), int(stop) - except ValueError: + position = int(location) + except (TypeError, ValueError): raise ValueError( - f"Error parsing position '{position}' extracted from GBFF string '{string}'. " - f"Start position ({start}) and/or stop position ({stop}) are not valid integers." + f"Gene position {location} is not formatted as expected. It currently is of type {type(location)}." ) - coordinates.append((start, stop)) - - return coordinates, complement, has_partial_start, has_partial_end - - -def parse_gbff_by_contig( - gbff_file_path: Path, -) -> Generator[ - Tuple[Dict[str, str], Generator[Dict[str, Union[str, Set[str]]], None, None], str], - None, - None, -]: - """ - Parse a GBFF file by contig and yield tuples containing header, feature, and sequence info for each contig. - - :param gbff_file_path: Path to the GBFF file. - :return: A generator that yields tuples containing header lines, feature lines, and sequence info for each contig. - """ - header_lines = [] - feature_lines = [] - sequence_lines = [] - - current_section = None - - with read_compressed_or_not(gbff_file_path) as fl: - for i, line in enumerate(fl): - # Skip blank lines - if not line.strip(): - continue - - if line.startswith("LOCUS") or line.startswith("CONTIG"): - # CONTIG line are found between FEATURES and ORIGIN and are put in header section here for simplicity - current_section = "header" - - elif line.startswith("FEATURES"): - current_section = "feature" - continue - - elif line.startswith("ORIGIN"): - current_section = "sequence" - continue - - if line.strip() == "//": - # Check that each section has some lines - assert header_lines and feature_lines and sequence_lines, ( - "Missing section in GBFF file. " - f"Contig ending at line {i + 1} has an empty section. It has " - f"{len(header_lines)} header lines, " - f"{len(header_lines)} feature lines, " - f"and {len(sequence_lines)} sequence lines." - ) - yield ( - parse_contig_header_lines(header_lines), - parse_feature_lines(feature_lines), - parse_dna_seq_lines(sequence_lines), - ) - - header_lines = [] - feature_lines = [] - sequence_lines = [] - current_section = None - continue - - if current_section == "header": - header_lines.append(line) - - elif current_section == "feature": - feature_lines.append(line) - - elif current_section == "sequence": - sequence_lines.append(line) - - else: - raise ValueError( - f"Unexpected structure in GBFF file: {gbff_file_path}. {line}" - ) - - # In case the last // is missing, return the last contig - if header_lines or feature_lines or sequence_lines: - yield ( - parse_contig_header_lines(header_lines), - parse_feature_lines(feature_lines), - parse_dna_seq_lines(sequence_lines), - ) - - -def parse_contig_header_lines(header_lines: List[str]) -> Dict[str, str]: - """ - Parse required information from header lines of a contig from a GBFF file. - - :param header_lines: List of strings representing header lines of a contig from a GBFF file. - :return: A dict with keys representing different fields and values representing their corresponding values joined by new line. - """ - field = "" # Initialize field - field_to_value = defaultdict( - list - ) # Initialize defaultdict to store field-value pairs - - for line in header_lines: - field_of_line = line[:12].strip() # Extract field from the first 12 characters - - if len(field_of_line) > 1 and field_of_line.isupper(): - field = field_of_line # Update current field - - # Append value to the current field in the defaultdict - field_to_value[field].append(line[12:].strip()) - - return {field: "\n".join(value) for field, value in field_to_value.items()} - - -def parse_feature_lines( - feature_lines: List[str], -) -> Generator[Dict[str, Union[str, Set[str]]], None, None]: - """ - Parse feature lines from a GBFF file and yield dictionaries representing each feature. - - :param feature_lines: List of strings representing feature lines from a GBFF file. - :return: A generator that yields dictionaries, each representing a feature with its type, location, and qualifiers. - """ - - def stringify_feature_values( - feature: Dict[str, List[str]] - ) -> Dict[str, Union[str, Set[str]]]: - """ - All value of the returned dict are str except for db_xref that is a list. - When multiple values exist for the same tag only the first one is kept. - """ - stringify_feature = {} - for tag, val in feature.items(): - if tag == "db_xref": - stringify_feature[tag] = set(val) - elif isinstance(val, list): - stringify_feature[tag] = val[0] - else: - stringify_feature[tag] = val - return defaultdict(str, stringify_feature) - - current_feature = {} - current_qualifier = None - - for line in feature_lines: - # Check if the line starts a new feature - if len(line[:21].strip()) > 0: - if current_feature: - # yield last feature - yield stringify_feature_values(current_feature) - - current_feature = { - "feature_type": line[:21].strip(), - "location": [line[21:].strip()], - } - current_qualifier = "location" - - elif line.strip().startswith("/"): - qualifier_line = line.strip()[1:] # [1:] used to remove / - - if "=" in qualifier_line: - current_qualifier, value = qualifier_line.split("=", 1) - else: - current_qualifier, value = qualifier_line, qualifier_line - # clean value from quote - value = value[1:] if value.startswith('"') else value - value = value[:-1] if value.endswith('"') else value - - if current_qualifier in current_feature: - current_feature[current_qualifier].append(value) - - else: - current_feature[current_qualifier] = [value] - - else: - # the line does not start a qualifier so it's the continuation of the last qualifier value. - value = line.strip() - value = value[:-1] if value.endswith('"') else value - current_feature[current_qualifier][-1] += f" {value}" + coordinates.append((position, position)) + parts = [] - # Append the last feature - if current_feature: - yield stringify_feature_values(current_feature) + for part in parts: + if not isinstance(part, gb_io.Range): + continue + has_partial_start |= getattr(part, "before", False) + has_partial_end |= getattr(part, "after", False) -def parse_dna_seq_lines(sequence_lines: List[str]) -> str: - """ - Parse sequence_lines from a GBFF file and return dna sequence + try: + coordinates.append((int(part.start) + 1, int(part.end))) + except (TypeError, ValueError): + raise ValueError( + f"Gene range {part} is not formatted as expected. It should contain integer start and end positions and before/after bools." + ) - :param sequence_lines: List of strings representing sequence lines from a GBFF file. - :return: a string in upper case of the DNA sequences that have been cleaned - """ - sequence = "" - for line in sequence_lines: - sequence += line[10:].replace(" ", "").strip().upper() - return sequence + return coordinates, is_complement, has_partial_start, has_partial_end def combine_contigs_metadata( - contig_to_metadata: Dict[Contig, Dict[str, str]] + contig_to_metadata: Dict[Contig, Dict[str, str]], ) -> Tuple[Dict[str, str], Dict[Contig, Dict[str, str]]]: """ Combine contig metadata to identify shared and unique metadata tags and values. @@ -473,7 +252,9 @@ def combine_contigs_metadata( invalid_tag_names.append(tag) all_tag_to_value = [ - (tag, value) for tag, value in all_tag_to_value if tag not in invalid_tag_names + (tag, value.replace("\n", " ")) + for tag, value in all_tag_to_value + if tag not in invalid_tag_names ] contig_count = len(contig_to_metadata) @@ -504,7 +285,7 @@ def combine_contigs_metadata( def reverse_complement_coordinates( - coordinates: List[Tuple[int, int]] + coordinates: List[Tuple[int, int]], ) -> List[Tuple[int, int]]: """ Reverses and inverts the given list of coordinates. Each coordinate pair (start, end) is transformed into @@ -642,24 +423,23 @@ def fix_partial_gene_coordinates( def read_org_gbff( organism_name: str, gbff_file_path: Path, - circular_contigs: List[str], + circular_contigs: list[str], use_pseudogenes: bool = False, -) -> Tuple[Organism, bool]: +) -> tuple[Organism, bool]: """ - Read a GBFF file and fills Organism, Contig and Genes objects based on information contained in this file + Read a GBFF file and fills Organism, Contig and Genes objects based on + information contained in this file :param organism_name: Organism name :param gbff_file_path: Path to corresponding GBFF file :param circular_contigs: list of contigs :param use_pseudogenes: Allow to read pseudogenes - :return: Organism complete and true for sequence in file """ global ctg_counter organism = Organism(organism_name) - logging.getLogger("PPanGGOLiN").debug( f"Extracting genes information from the given gbff {gbff_file_path.name}" ) @@ -667,153 +447,144 @@ def read_org_gbff( rna_counter = 0 contig_to_metadata = {} - for header, features, sequence in parse_gbff_by_contig(gbff_file_path): - if "LOCUS" not in header: - raise ValueError("Missing LOCUS line in GBFF header.") + with read_compressed_or_not(gbff_file_path) as reader: + for record in gb_io.iter(reader): - if "VERSION" in header and header["VERSION"] != "": - contig_id = header["VERSION"] - else: - # If contig_id is not specified in VERSION field like with Prokka, in that case we use the one in LOCUS - contig_id = header["LOCUS"].split()[0] + contig_id = record.version if record.version else record.name + contig_len = record.length + is_circ = record.circular - contig_len = int(header["LOCUS"].split()[1]) + try: + contig = organism.get(contig_id) + except KeyError: + with contig_counter.get_lock(): + contig = Contig( + contig_counter.value, + contig_id, + True if contig_id in circular_contigs or is_circ else False, + ) + contig_counter.value += 1 + organism.add(contig) + contig.length = contig_len + + for feature in record.features: + + db_xref_for_metadata = {} + + if feature.kind == "source": + for qualifier in feature.qualifiers: + + if qualifier.key == "db_xref": + try: + database, identifier = qualifier.value.split(":") + new_key = qualifier.key + "_" + database + db_xref_for_metadata[new_key] = identifier + except ValueError: + logging.getLogger("PPanGGOLiN").warning( + f"db_xref values does not have the expected format. Expect 'db_xref=:' " + f"but got {qualifier.value} in file {gbff_file_path}. " + "db_xref tags is therefore not retrieved in " + "contig/genomes metadata." + ) + contig_to_metadata[contig] = { + qualifier.key: qualifier.value + for qualifier in feature.qualifiers + if isinstance(qualifier.value, str) + and qualifier.key != "db_xref" + } - if contig_len != len(sequence): - logging.getLogger("PPanGGOLiN").warning( - "Unable to determine if the contig is circular or linear in file " - f"'{gbff_file_path}' from the LOCUS header information: {header['LOCUS']}. " - "By default, the contig will be considered linear." - ) + contig_to_metadata[contig].update(db_xref_for_metadata) - if "CIRCULAR" in header["LOCUS"].upper(): - # this line contains linear/circular word telling if the dna sequence is circularized or not - is_circ = True + genetic_code = "" - elif "LINEAR" in header["LOCUS"].upper(): - is_circ = False + if feature.kind not in ["CDS", "rRNA", "tRNA"]: + continue - else: - is_circ = False - logging.getLogger("PPanGGOLiN").warning( - f"It's impossible to identify if contig {contig_id} is circular or linear." - f"in file {gbff_file_path}." - ) + coordinates, is_complement, has_partial_start, has_partial_end = ( + extract_positions(feature.location) + ) - try: - contig = organism.get(contig_id) - except KeyError: - with contig_counter.get_lock(): - contig = Contig( - contig_counter.value, - contig_id, - True if contig_id in circular_contigs or is_circ else False, + feature_qualifiers = defaultdict( + str, + { + qualifier.key: qualifier.value + for qualifier in feature.qualifiers + }, ) - contig_counter.value += 1 - organism.add(contig) - contig.length = contig_len - - for feature in features: - if feature["feature_type"] == "source": - contig_to_metadata[contig] = { - tag: value - for tag, value in feature.items() - if tag not in ["feature_type", "location"] - and isinstance(value, str) - } - if "db_xref" in feature: - try: - db_xref_for_metadata = { - f"db_xref_{database}": identifier - for database_identifier in feature["db_xref"] - for database, identifier in [database_identifier.split(":")] - } - contig_to_metadata[contig].update(db_xref_for_metadata) - except ValueError: - logging.getLogger("PPanGGOLiN").warning( - f"db_xref values does not have the expected format. Expect 'db_xref=:' " - f"but got {feature['db_xref']} in file {gbff_file_path}. " - "db_xref tags is therefore not retrieved in contig/genomes metadata." - ) - else: - contig_to_metadata[contig].update(db_xref_for_metadata) - genetic_code = "" - if feature["feature_type"] not in ["CDS", "rRNA", "tRNA"]: - continue - coordinates, is_complement, has_partial_start, has_partial_end = ( - extract_positions("".join(feature["location"])) - ) - if "pseudo" in feature and not use_pseudogenes: - continue + if "pseudo" in feature_qualifiers and not use_pseudogenes: + continue - elif "transl_except" in feature and not use_pseudogenes: - # that's probably a 'stop' codon into selenocystein. - logging.getLogger("PPanGGOLiN").info( - f"CDS '{feature['locus_tag']}' contains a 'transl_except' annotation ({feature['transl_except']}) " - f"in contig '{contig}' in file '{gbff_file_path}'. " - f"PPanGGOLiN does not handle 'transl_except' annotations. This gene's protein sequence " - "will likely contain an internal stop codon when translated with PPanGGOLiN." - ) + elif "transl_except" in feature_qualifiers and not use_pseudogenes: + # that's probably a 'stop' codon into selenocystein. + logging.getLogger("PPanGGOLiN").info( + f"CDS '{feature['locus_tag']}' contains a 'transl_except' annotation ({feature['transl_except']}) " + f"in contig '{contig}' in file '{gbff_file_path}'. " + f"PPanGGOLiN does not handle 'transl_except' annotations. This gene's protein sequence " + "will likely contain an internal stop codon when translated with PPanGGOLiN." + ) - for field in ["product", "gene", "db_xref"]: + for field in ["product", "gene", "db_xref"]: + if field in feature_qualifiers and has_non_ascii( + feature_qualifiers[field] + ): + logging.getLogger("PPanGGOLiN").warning( + f"In genome '{organism}', the '{field}' field of gene '{feature_qualifiers['locus_tag']}' contains non-ASCII characters: '{feature_qualifiers[field]}'. " + "These characters cannot be stored in the HDF5 file and will be replaced by underscores." + ) - if field in feature and has_non_ascii(feature[field]): + feature_qualifiers[field] = replace_non_ascii( + feature_qualifiers[field] + ) - logging.getLogger("PPanGGOLiN").warning( - f"In genome '{organism}', the '{field}' field of gene '{feature['locus_tag']}' contains non-ASCII characters: '{feature[field]}'. " - "These characters cannot be stored in the HDF5 file and will be replaced by underscores." - ) - feature[field] = replace_non_ascii(feature[field]) - - if feature["feature_type"] == "CDS": - genetic_code = 0 - if feature["transl_table"] != "": - genetic_code = int(feature["transl_table"]) - - if has_partial_start or has_partial_end: - start_shift = ( - 0 - if "codon_start" not in feature - else int(feature["codon_start"]) - 1 - ) # -1 is to be in zero based index. - - coordinates = fix_partial_gene_coordinates( - coordinates, - is_complement=is_complement, - start_shift=start_shift, - ) + if feature.kind == "CDS": + genetic_code = 0 + if feature_qualifiers.get("transl_table", "") != "": + genetic_code = int(feature_qualifiers["transl_table"]) + + if has_partial_start or has_partial_end: + start_shift = ( + 0 + if "codon_start" not in feature_qualifiers + else int(feature_qualifiers["codon_start"]) - 1 + ) # -1 is to be in zero based index. + + coordinates = fix_partial_gene_coordinates( + coordinates, + is_complement=is_complement, + start_shift=start_shift, + ) - strand = "-" if is_complement else "+" - - gene = create_gene( - org=organism, - contig=contig, - gene_counter=gene_counter, - rna_counter=rna_counter, - gene_id=feature["locus_tag"], - dbxrefs=feature["db_xref"], - coordinates=coordinates, - strand=strand, - gene_type=feature["feature_type"], - position=contig.number_of_genes, - gene_name=feature["gene"], - product=feature["product"], - genetic_code=genetic_code, - protein_id=feature["protein_id"], - ) + strand = "-" if is_complement else "+" + gene = create_gene( + org=organism, + contig=contig, + gene_counter=gene_counter, + rna_counter=rna_counter, + gene_id=feature_qualifiers["locus_tag"], + dbxrefs=[feature_qualifiers["db_xref"]], + coordinates=coordinates, + strand=strand, + gene_type=feature.kind, + position=contig.number_of_genes, + gene_name=feature_qualifiers["gene"], + product=feature_qualifiers["product"].replace("\n", " "), + genetic_code=genetic_code, + protein_id=feature_qualifiers["protein_id"], + ) - gene.add_sequence(get_dna_sequence(sequence, gene)) + gene.add_sequence( + get_dna_sequence(record.sequence.decode("ascii").upper(), gene) + ) - if feature["feature_type"] == "CDS": - gene_counter += 1 - else: - rna_counter += 1 + if feature.kind == "CDS": + gene_counter += 1 + else: + rna_counter += 1 genome_metadata, contig_to_uniq_metadata = combine_contigs_metadata( contig_to_metadata ) - if genome_metadata: organism.add_metadata( metadata=Metadata(source="annotation_file", **genome_metadata) @@ -898,7 +669,7 @@ def get_gff_attributes(gff_fields: list) -> dict: attributes_get = {} for att in attributes_field: try: - (key, value) = att.strip().split("=") + key, value = att.strip().split("=") attributes_get[key.upper()] = value except ValueError: pass # we assume that it is a strange, but useless field for our analysis @@ -1624,20 +1395,10 @@ def read_annotations( pangenome.status["geneSequences"] = "Computed" # we assume there are gene sequences in the annotation files, # unless a gff file without fasta is met (which is the only case where sequences can be absent) + genome_paths = parse_input_paths_file(organisms_file) args = [] - for line in read_compressed_or_not(organisms_file): - if not line.strip() or line.strip().startswith("#"): - continue - elements = [el.strip() for el in line.split("\t")] - org_path = Path(elements[1]) - name = elements[0] - circular_contigs = elements[2:] - if ( - not org_path.exists() - ): # Check tsv sanity test if it's not one it's the other - org_path = organisms_file.parent.joinpath(org_path) - - args.append((name, org_path, circular_contigs, pseudo)) + for name, path_info in genome_paths.items(): + args.append((name, path_info["path"], path_info["circular_contigs"], pseudo)) with ProcessPoolExecutor( mp_context=get_context("fork"), @@ -1717,22 +1478,17 @@ def get_gene_sequences_from_fastas( :param disable_bar: Flag to disable progress bar """ fasta_dict = {} - for line in read_compressed_or_not(fasta_files): - elements = [el.strip() for el in line.split("\t")] - if len(elements) <= 1: - logging.getLogger("PPanGGOLiN").error( - "No tabulation separator found in genome file" - ) - exit(1) + genome_paths = parse_input_paths_file(fasta_files) + for genome_name, path_info in genome_paths.items(): try: - org = pangenome.get_organism(elements[0]) + org = pangenome.get_organism(genome_name) except KeyError: raise KeyError( f"One of the genome in your '{fasta_files}' was not found in the pan." f" This might mean that the genome names between your annotation file and " f"your fasta file are different." ) - with read_compressed_or_not(Path(elements[1])) as currFastaFile: + with read_compressed_or_not(path_info["path"]) as currFastaFile: fasta_dict[org] = get_contigs_from_fasta_file(org, currFastaFile) # When dealing with GFF files, some genes may have coordinates extending beyond the actual @@ -1821,22 +1577,14 @@ def annotate_pangenome( f"Reading {fasta_list} the list of genome files" ) + genome_paths = parse_input_paths_file(fasta_list) arguments = [] # Argument given to annotate organism in same order than prototype - for line in read_compressed_or_not(fasta_list): - - elements = [el.strip() for el in line.split("\t")] - org_path = Path(elements[1]) - - if ( - not org_path.exists() - ): # Check tsv sanity test if it's not one it's the other - org_path = fasta_list.parent.joinpath(org_path) - + for genome_name, path_info in genome_paths.items(): arguments.append( ( - elements[0], - org_path, - elements[2:], + genome_name, + path_info["path"], + path_info["circular_contigs"], tmpdir, translation_table, norna, @@ -1846,9 +1594,6 @@ def annotate_pangenome( ) ) - if len(arguments) == 0: - raise Exception("There are no genomes in the provided file") - logging.getLogger("PPanGGOLiN").info( f"Annotating {len(arguments)} genomes using {cpu} cpus..." ) @@ -1975,6 +1720,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "annotate", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Annotate genomes" + parser.category = "Expert" parser_annot(parser) return parser @@ -1994,16 +1741,15 @@ def parser_annot(parser: argparse.ArgumentParser): "--fasta", required=False, type=Path, - help="A tab-separated file listing the genome names, and the fasta filepath of its genomic " - "sequence(s) (the fastas can be compressed with gzip). One line per genome.", + help="A tab-separated file listing genome names and the FASTA file paths of their genomic sequences " + "(the FASTA files can be compressed with gzip). One line per genome.", ) required.add_argument( "--anno", required=False, type=Path, - help="A tab-separated file listing the genome names, and the gff/gbff filepath of its " - "annotations (the files can be compressed with gzip). One line per genome. " - "If this is provided, those annotations will be used.", + help="A tab-separated file listing genome names and the GFF/GBFF file paths of their annotations " + "(the files can be compressed with gzip). One line per genome. If this is provided, those annotations will be used.", ) optional = parser.add_argument_group(title="Optional arguments") @@ -2013,21 +1759,21 @@ def parser_annot(parser: argparse.ArgumentParser): required=False, type=Path, default=Path(f"ppanggolin_output{date}_PID{str(os.getpid())}"), - help="Output directory", + help="Output directory.", ) optional.add_argument( "--allow_overlap", required=False, action="store_true", default=False, - help="Use to not remove genes overlapping with RNA features.", + help="Use this option to keep genes overlapping with RNA features.", ) optional.add_argument( "--norna", required=False, action="store_true", default=False, - help="Use to avoid annotating RNA features.", + help="Use this option to avoid annotating RNA features.", ) optional.add_argument( "--kingdom", @@ -2035,8 +1781,7 @@ def parser_annot(parser: argparse.ArgumentParser): type=str.lower, default="bacteria", choices=["bacteria", "archaea"], - help="Kingdom to which the prokaryota belongs to, " - "to know which models to use for rRNA annotation.", + help="Kingdom to which the prokaryote belongs, to determine which models to use for rRNA annotation.", ) optional.add_argument( "--translation_table", @@ -2052,14 +1797,14 @@ def parser_annot(parser: argparse.ArgumentParser): "--basename", required=False, default="pangenome", - help="basename for the output file", + help="Basename for the output file.", ) optional.add_argument( "--use_pseudo", required=False, action="store_true", help="In the context of provided annotation, use this option to read pseudogenes. " - "(Default behavior is to ignore them)", + "(Default behavior is to ignore them).", ) optional.add_argument( "-p", @@ -2068,8 +1813,7 @@ def parser_annot(parser: argparse.ArgumentParser): type=str.lower, choices=["single", "meta"], default=None, - help="Allow to force the prodigal procedure. " - "If nothing given, PPanGGOLiN will decide in function of contig length", + help="Allow forcing the Prodigal procedure. If not specified, PPanGGOLiN will decide based on contig length.", ) optional.add_argument( "-c", @@ -2077,14 +1821,14 @@ def parser_annot(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) optional.add_argument( "--tmpdir", required=False, type=str, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) diff --git a/ppanggolin/annotate/synta.py b/ppanggolin/annotate/synta.py index 44353090..a9227c21 100644 --- a/ppanggolin/annotate/synta.py +++ b/ppanggolin/annotate/synta.py @@ -7,7 +7,6 @@ from io import TextIOWrapper from multiprocessing import Value from subprocess import Popen, PIPE -import ast from collections import defaultdict, Counter from typing import Dict, List, Optional, Union, Generator, Tuple from pathlib import Path @@ -15,6 +14,7 @@ # install libraries from pyrodigal import GeneFinder, Sequence +from pyaragorn import RNAFinder # local libraries from ppanggolin.genome import Organism, Gene, RNA, Contig @@ -40,7 +40,7 @@ def reverse_complement(seq: str): :return: reverse sequence """ - + seq = seq.upper() complement = { "A": "T", "C": "G", @@ -66,64 +66,55 @@ def reverse_complement(seq: str): def launch_aragorn( - fna_file: str, org: Organism, contig_to_length: Dict[str, int] + contig_sequences: Dict[str, str], org: Organism, contig_to_length: Dict[str, int] ) -> defaultdict: """ - Launches Aragorn to annotate tRNAs. + Launches Aragorn to annotate tRNAs using pyaragorn. - :param fna_file: file-like object containing the uncompressed fasta sequences + :param contig_sequences: Dict containing contig sequences :param org: Organism which will be annotated + :param contig_to_length: Dict mapping contig names to their lengths :return: Annotated genes in a list of gene objects """ locustag = org.name - cmd = ["aragorn", "-t", "-gcbact", "-l", "-w", fna_file] - logging.getLogger("PPanGGOLiN").debug(f"aragorn command : {' '.join(cmd)}") - - if shutil.which("aragorn") is None: - raise FileNotFoundError( - f"Command 'aragorn' not found. Please install it and try again." - ) + rna_finder = RNAFinder( + translation_table=11, + trna=True, + tmrna=False, + linear=True, + ) - p = Popen(cmd, stdout=PIPE) - # loading the whole thing, reverting it to 'pop' in order. - file_data = p.communicate()[0].decode().split("\n")[::-1] gene_objs = defaultdict(set) c = 0 - contig_name = "" - while len(file_data) != 0: - line = file_data.pop() - if line.startswith(">"): - contig_name = line.replace(">", "").split()[0] - file_data.pop() # then next line must be removed too. - elif len(line) > 0: # if the line isn't empty, there's data to get. - line_data = line.split() - start, stop = map(int, ast.literal_eval(line_data[2].replace("c", ""))) + for contig_name, sequence in contig_sequences.items(): + contig_length = contig_to_length[contig_name] + genes = rna_finder.find_rna(sequence) + for g in genes: + start = g.begin + stop = g.end if start < 1 or stop < 1: - # In some case aragorn gives negative coordinates. This case is just ignored. + # In some case aragorn gives non valid coordinates. This case is just ignored. logging.warning( - f"Aragorn gives non valid coordinates for a RNA gene in contig {contig_name}: {line_data}. This RNA is ignored." + f"Aragorn gives non valid coordinates for a RNA gene in contig {contig_name}: " + f"begin={g.begin} end={g.end} strand={g.strand}. This RNA is ignored." ) continue - if ( - start > contig_to_length[contig_name] - or stop > contig_to_length[contig_name] - ): + if start > contig_length or stop > contig_length: logging.warning( f"Aragorn gives non valide coordinates for a RNA gene in contig {contig_name}. " - f"Gene coordinates exceed contig length ({contig_to_length[contig_name]}): " - f"{line_data}. This RNA is ignored." + f"Gene coordinates exceed contig length ({contig_length}): " + f"begin={g.begin} end={g.end}. This RNA is ignored." ) continue - c += 1 gene = RNA(rna_id=locustag + "_tRNA_" + str(c).zfill(4)) gene.fill_annotations( start=start, stop=stop, - strand="-" if line_data[2].startswith("c") else "+", - gene_type="tRNA", - product=line_data[1] + line_data[4], + strand="-" if g.strand == -1 else "+", + gene_type=g.type, + product=f"{g.type}-{g.amino_acid}({g.anticodon})", ) gene_objs[contig_name].add(gene) return gene_objs @@ -278,32 +269,8 @@ def launch_infernal( return gene_objs -def check_sequence_tuple(name: str, sequence: str): - """ - Checks and validates a sequence name and its corresponding sequence. - - :param name: The name (header) of the sequence, typically extracted from the FASTA file header. - :param sequence: The sequence string corresponding to the name, containing the nucleotide or protein sequence. - - :return: A tuple containing the validated name and sequence. - - :raises ValueError: - - If the sequence is empty, a ValueError is raised with a message containing the header name. - - If the name is empty, a ValueError is raised with a message containing a preview of the sequence. - """ - if not sequence: - raise ValueError(f"Found an empty sequence with header '{name}'") - - if not name: - raise ValueError( - f"Found a sequence with empty name (sequence starts as '{sequence[:60]}')" - ) - - return name, sequence - - def parse_fasta( - fna_file: Union[TextIOWrapper, list] + fna_file: Union[TextIOWrapper, list], ) -> Generator[Tuple[str, str], None, None]: """Yields each sequence name and sequence from a FASTA file or stream as a tuple. @@ -311,35 +278,98 @@ def parse_fasta( :yield: Tuple with contig header (without '>') and sequence. :raises ValueError: If the file does not contain valid FASTA format. """ + source_name = getattr(fna_file, "name", None) + + def validate_sequence(name: str, sequence: str): + if not name: + if source_name: + raise ValueError( + f"Malformed FASTA header in '{source_name}': a contig name is empty. " + "Each FASTA header must start with '>' and contain a contig name." + ) + raise ValueError( + "Malformed FASTA header: a contig name is empty. " + "Each FASTA header must start with '>' and contain a contig name." + ) + + if not sequence: + if source_name: + raise ValueError( + f"Malformed FASTA record for contig '{name}' in '{source_name}': the sequence is empty. " + "Each contig entry must contain at least one nucleotide or amino-acid base." + ) + raise ValueError( + f"Malformed FASTA record for contig '{name}': the sequence is empty. " + "Each contig entry must contain at least one nucleotide or amino-acid base." + ) + + return name, sequence + name = None sequence = "" + saw_header = False - for line in fna_file: + for line_number, line in enumerate(fna_file, start=1): line = line.strip() - if line.startswith(">"): # New header - if name: # Yield previous header and sequence if available - yield check_sequence_tuple(name, sequence) + if not line: + continue - name = line[1:].split()[ - 0 - ] # Strip '>' and extract the first word as the name + if line.startswith(">"): + if name is not None: + yield validate_sequence(name, sequence) + raw_name = line[1:].strip() + if not raw_name: + if source_name: + raise ValueError( + f"Malformed FASTA header in '{source_name}' on line {line_number}: header is empty. " + "Each record must start with '>' followed by a contig name." + ) + raise ValueError( + f"Malformed FASTA header on line {line_number}: header is empty. " + "Each record must start with '>' followed by a contig name." + ) + name = raw_name.split()[0] + if not name: + if source_name: + raise ValueError( + f"Malformed FASTA header in '{source_name}' on line {line_number}: the contig name is empty or blank. " + "Please provide a valid header such as '>contig_1'." + ) + raise ValueError( + f"Malformed FASTA header on line {line_number}: the contig name is empty or blank. " + "Please provide a valid header such as '>contig_1'." + ) + saw_header = True sequence = "" + continue - elif line: # Only append non-empty lines - sequence += line + if name is None: + if source_name: + raise ValueError( + f"Malformed FASTA content in '{source_name}' on line {line_number}: found sequence data before any '>' header. " + "FASTA records must begin with a header such as '>contig_1'." + ) + raise ValueError( + f"Malformed FASTA content on line {line_number}: found sequence data before any '>' header. " + "FASTA records must begin with a header such as '>contig_1'." + ) - else: - # You can skip or handle empty lines here if required - pass + sequence += line - # Yield the final contig if exists - if name: - yield check_sequence_tuple(name, sequence) + if name is not None: + yield validate_sequence(name, sequence) - # Check if there was any valid data (at least one header and sequence) - if not name: - raise ValueError("The file does not contain any valid FASTA content.") + if not saw_header: + if source_name: + raise ValueError( + f"The file '{source_name}' does not contain any valid FASTA content. " + "Expected at least one header line starting with '>'." + ) + raise ValueError( + f"The file '{source_name}' does not contain any valid FASTA content. " + "Expected at least one header line starting with '>'." + ) def get_contigs_from_fasta_file( @@ -438,7 +468,9 @@ def syntaxic_annotation( } for contig_name, genes_from_contig in launch_aragorn( - fna_file=fasta_file.name, org=org, contig_to_length=contig_to_length + contig_sequences=contig_sequences, + org=org, + contig_to_length=contig_to_length, ).items(): genes[contig_name].extend(genes_from_contig) for contig_name, genes_from_contig in launch_infernal( @@ -549,7 +581,7 @@ def annotate_organism( """ if not norna: - check_tools_availability({"aragorn": "", "cmscan": " from the tool Infernal"}) + check_tools_availability({"cmscan": " from the tool Infernal"}) org = Organism(org_name) diff --git a/ppanggolin/cluster/cluster.py b/ppanggolin/cluster/cluster.py index 1b01e674..7b5e34b5 100644 --- a/ppanggolin/cluster/cluster.py +++ b/ppanggolin/cluster/cluster.py @@ -909,6 +909,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "cluster", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Cluster genes into gene families" + parser.category = "Expert" parser_clust(parser) return parser @@ -924,7 +926,11 @@ def parser_clust(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) clust = parser.add_argument_group(title="Clustering arguments") clust.add_argument( @@ -932,30 +938,29 @@ def parser_clust(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.8, - help="Minimal identity percent for two proteins to be in the same cluster", + help="Minimum identity percentage for two proteins to be in the same cluster.", ) clust.add_argument( "--coverage", required=False, type=restricted_float, default=0.8, - help="Minimal coverage of the alignment for two proteins to be in the same cluster", + help="Minimum coverage of the alignment for two proteins to be in the same cluster.", ) clust.add_argument( "--mode", required=False, default="1", choices=["0", "1", "2", "3"], - help="the cluster mode of MMseqs2. 0: Setcover, 1: single linkage (or connected component)," - " 2: CD-HIT-like, 3: CD-HIT-like (lowmem)", + help="MMseqs2 clustering mode: 0 = Setcover, 1 = single linkage (or connected component), " + "2 = CD-HIT-like, 3 = CD-HIT-like (lowmem).", ) clust.add_argument( "--no_defrag", required=False, default=False, action="store_true", - help="DO NOT Use the defragmentation strategy to link potential fragments " - "with their original gene family.", + help="Do not use the defragmentation strategy to link potential fragments with their original gene family.", ) read = parser.add_argument_group(title="Read clustering arguments") @@ -964,14 +969,14 @@ def parser_clust(parser: argparse.ArgumentParser): required=False, type=Path, help="A tab-separated list containing the result of a clustering. One line per gene. " - "First column is cluster ID, and second is gene ID", + "The first column is the cluster ID and the second is the gene ID.", ) read.add_argument( "--infer_singletons", required=False, action="store_true", - help="When reading a clustering result with --clusters, if a gene is not in the provided file" - " it will be placed in a cluster where the gene is the only member.", + help="When reading a clustering result with --clusters, if a gene is not in the provided file, " + "it will be placed in a cluster where the gene is the only member.", ) optional = parser.add_argument_group(title="Optional arguments") optional.add_argument( @@ -989,21 +994,21 @@ def parser_clust(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) optional.add_argument( "--tmpdir", required=False, type=Path, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) optional.add_argument( "--keep_tmp", required=False, default=False, action="store_true", - help="Keeping temporary files (useful for debugging).", + help="Keep temporary files (useful for debugging).", ) diff --git a/ppanggolin/context/searchGeneContext.py b/ppanggolin/context/searchGeneContext.py index 543a91ff..d5a41dd0 100644 --- a/ppanggolin/context/searchGeneContext.py +++ b/ppanggolin/context/searchGeneContext.py @@ -809,6 +809,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "context", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Local genomic context analysis." + parser.category = "Analysis using reference pangenomes" parser_context(parser) return parser @@ -825,7 +827,11 @@ def parser_context(parser: argparse.ArgumentParser): description="All of the following arguments are required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome.h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) required.add_argument( "-o", @@ -836,24 +842,24 @@ def parser_context(parser: argparse.ArgumentParser): + time.strftime("_DATE%Y-%m-%d_HOUR%H.%M.%S", time.localtime()) + "_PID" + str(os.getpid()), - help="Output directory where the file(s) will be written", + help="Output directory where the file(s) will be written.", ) onereq = parser.add_argument_group( - title="Input file", description="One of the following argument is required :" + title="Input file", description="One of the following arguments is required:" ) onereq.add_argument( "-S", "--sequences", required=False, type=Path, - help="Fasta file with the sequences of interest", + help="FASTA file with the sequences of interest.", ) onereq.add_argument( "-F", "--family", required=False, type=Path, - help="List of family IDs of interest from the pangenome", + help="List of family IDs of interest from the pangenome.", ) optional = parser.add_argument_group(title="Optional arguments") optional.add_argument( @@ -863,8 +869,7 @@ def parser_context(parser: argparse.ArgumentParser): type=int, default=4, help="Size of the transitive closure used to build the graph. This indicates the number of " - "non related genes allowed in-between two related genes. Increasing it will improve " - "precision but lower sensitivity a little.", + "unrelated genes allowed between two related genes. Increasing it will improve precision but lower sensitivity slightly.", ) optional.add_argument( "-w", @@ -872,8 +877,7 @@ def parser_context(parser: argparse.ArgumentParser): required=False, type=int, default=5, - help="Number of neighboring genes that are considered on each side of " - "a gene of interest when searching for conserved genomic contexts.", + help="Number of neighboring genes considered on each side of a gene of interest when searching for conserved genomic contexts.", ) optional.add_argument( @@ -882,25 +886,23 @@ def parser_context(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.85, - help="minimum jaccard similarity used to filter edges between gene families. Increasing it " - "will improve precision but lower sensitivity a lot.", + help="Minimum Jaccard similarity used to filter edges between gene families. Increasing it will improve precision but lower sensitivity considerably.", ) optional.add_argument( "--graph_format", - help="Format of the context graph. Can be gexf or graphml.", + help="Format of the context graph. Can be GEXF or GraphML.", default="graphml", choices=["gexf", "graphml"], ) align = parser.add_argument_group( title="Alignment arguments", - description="This argument makes sense only when --sequence is provided.", + description="This argument only makes sense when --sequence is provided.", ) align.add_argument( "--no_defrag", required=False, action="store_true", - help="DO NOT Realign gene families to link fragments with" - "their non-fragmented gene family.", + help="Do not realign gene families to link fragments with their non-fragmented gene family.", ) align.add_argument( "--fast", @@ -915,14 +917,14 @@ def parser_context(parser: argparse.ArgumentParser): required=False, type=float, default=0.8, - help="min identity percentage threshold", + help="Minimum identity percentage threshold.", ) align.add_argument( "--coverage", required=False, type=float, default=0.8, - help="min coverage percentage threshold", + help="Minimum coverage percentage threshold.", ) align.add_argument( "--translation_table", @@ -938,14 +940,14 @@ def parser_context(parser: argparse.ArgumentParser): required=False, type=str, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) align.add_argument( "--keep_tmp", required=False, default=False, action="store_true", - help="Keeping temporary files (useful for debugging).", + help="Keep temporary files (useful for debugging).", ) align.add_argument( "-c", @@ -953,7 +955,7 @@ def parser_context(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) diff --git a/ppanggolin/figures/drawing.py b/ppanggolin/figures/drawing.py index 46c67447..eaf161d1 100644 --- a/ppanggolin/figures/drawing.py +++ b/ppanggolin/figures/drawing.py @@ -83,6 +83,10 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "draw", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = ( + "Draw figures representing the pangenome through different aspects" + ) + parser.category = "Output" parser_draw(parser) return parser @@ -99,7 +103,11 @@ def parser_draw(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome.h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) optional = parser.add_argument_group(title="Optional arguments") @@ -109,7 +117,7 @@ def parser_draw(parser: argparse.ArgumentParser): required=False, type=Path, default=Path(f"ppanggolin_output{date}_PID{str(os.getpid())}"), - help="Output directory", + help="Output directory.", ) optional.add_argument( "--tile_plot", diff --git a/ppanggolin/formats/h5reader.py b/ppanggolin/formats/h5reader.py new file mode 100644 index 00000000..b7efe574 --- /dev/null +++ b/ppanggolin/formats/h5reader.py @@ -0,0 +1,175 @@ +import tables +from pathlib import Path +from dataclasses import dataclass +import typing as tp + + +@dataclass +class TableAttribute: + """Describe how a table column should be read and converted. + + :param name: Name of the HDF5 column in the table. + :param predicate: Optional filter used to decide whether a row should be kept. + :param transform: Optional callable used to transform the raw value before instantiation. + """ + + name: str + predicate: tp.Callable = lambda x: True + transform: tp.Callable = lambda x: x + + +def get_table_attributes(cls): + """Collect table column metadata annotations from a dataclass. + + :param cls: Dataclass type describing the HDF5 table schema. + :return: Mapping of field names to their TableAttribute metadata. + """ + attrs = {} + for field_name, annotated_type in tp.get_type_hints( + cls, include_extras=True + ).items(): + if tp.get_origin(annotated_type) is tp.Annotated: + _, table_attr = tp.get_args(annotated_type) + attrs[field_name] = table_attr + return attrs + + +class H5Reader: + """Lightweight HDF5 table reader for PPanGGOLiN-style dataclass mappings.""" + + def __init__(self, h5_file: Path) -> None: + """Initialize the reader for an HDF5 file. + + :param h5_file: Path to the pangenome HDF5 file. + :raises FileNotFoundError: If the HDF5 file does not exist. + """ + if not h5_file.exists(): + raise FileNotFoundError(f"The file {h5_file} does not exist.") + self.h5_file = h5_file + self.handle: tables.File | None = None + + def open(self) -> None: + """Open the underlying HDF5 file in read mode.""" + self.handle = tables.open_file(str(self.h5_file), mode="r") + + def close(self) -> None: + """Close the HDF5 file if it is currently open.""" + if self.handle: + self.handle.close() + self.handle = None + + def __enter__(self) -> "H5Reader": + """Open the reader when used as a context manager. + + :return: The initialized H5Reader instance. + """ + self.open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + """Close the reader when leaving a context manager block.""" + self.close() + + def _check(self, obj: tp.Any) -> dict[str, TableAttribute]: + """Validate that a dataclass exposes a readable HDF5 table definition. + + :param obj: Dataclass type describing the table schema. + :raises ValueError: If the schema is missing required metadata or the table is absent. + :return: The table name and column metadata mapping. + """ + attrs = get_table_attributes(obj) + if not attrs: + raise ValueError(f"No TableAttributes found in {obj.__name__}.") + if "_table" not in obj.__dict__: + raise ValueError( + f'Missing required TableAttribute "_table" in {obj.__name__}.' + ) + + table_name = obj._table + + if table_name not in self.handle: + raise ValueError( + f"Required table '{table_name}' is missing from the pangenome H5 file: {self.h5_file}" + ) + + return table_name, attrs + + def _apply_override(self, attrs, override): + """Apply row-level overrides to a table attribute mapping. + + :param attrs: Mapping of fields to TableAttribute definitions. + :param override: Optional per-field overrides with transform and/or predicate values. + """ + if not override: + return + + for field_name, attr in attrs.items(): + if field_name in override: + if "transform" in override[field_name]: + attr.transform = override[field_name]["transform"] + if "predicate" in override[field_name]: + attr.predicate = override[field_name]["predicate"] + + def fetch( + self, obj: tp.Any, override: dict = {} + ) -> tp.Generator[tp.Any, None, None]: + """Yield dataclass instances generated from rows in the matching HDF5 table. + + :param obj: Dataclass type used to construct each row object. + :param override: Optional field overrides for transform and predicate hooks. + :raises RuntimeError: If the HDF5 file is not open. + :return: Generator of instantiated dataclass objects. + """ + if self.handle is None: + raise RuntimeError( + "H5 file is not opened. Use 'with' statement or call open() method." + ) + + table_name, attrs = self._check(obj) + + h5_table = self.handle.get_node(table_name) + + self._apply_override(attrs, override) + + for row in h5_table.iterrows(): + args = {} + for field_name, attr in attrs.items(): + value = row[attr.name] + if not attr.predicate(value): + break + else: + args[field_name] = attr.transform(value) + else: + yield obj(**args) + + def fetch_raw(self, obj: tp.Any) -> tp.Generator[tp.Any, None, None]: + """Yield raw table rows as dataclass instances without applying row filtering. + + :param obj: Dataclass type used to construct each object. + :raises RuntimeError: If the HDF5 file is not open. + :return: Generator of dataclass objects with raw column values. + """ + if self.handle is None: + raise RuntimeError( + "H5 file is not opened. Use 'with' statement or call open() method." + ) + + table_name, attrs = self._check(obj) + h5_table = self.handle.get_node(table_name) + + for row in h5_table.iterrows(): + obj(**{field_name: row[attr.name] for field_name, attr in attrs.items()}) + + def fetch_rows(self, obj: tp.Any) -> tp.Generator[tp.Any, None, None]: + """Yield the raw row objects stored in a table without object conversion. + + :param obj: Dataclass type describing the table schema. + :raises RuntimeError: If the HDF5 file is not open. + :return: Generator of raw rows from the selected HDF5 table. + """ + if self.handle is None: + raise RuntimeError( + "H5 file is not opened. Use 'with' statement or call open() method." + ) + table_name, _ = self._check(obj) + yield from self.handle.get_node(table_name).iterrows() diff --git a/ppanggolin/formats/writeFlatGenomes.py b/ppanggolin/formats/writeFlatGenomes.py index f9f7a9c1..5f7c0cc3 100644 --- a/ppanggolin/formats/writeFlatGenomes.py +++ b/ppanggolin/formats/writeFlatGenomes.py @@ -859,6 +859,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "write_genomes", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Writes 'flat' files that represent the genomes along with their associated pangenome elements." + parser.category = "Output" parser_flat(parser) return parser @@ -874,14 +876,18 @@ def parser_flat(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) required.add_argument( "-o", "--output", required=True, type=Path, - help="Output directory where the file(s) will be written", + help="Output directory where the file(s) will be written.", ) optional = parser.add_argument_group(title="Optional arguments") @@ -954,7 +960,7 @@ def parser_flat(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) context = parser.add_argument_group( @@ -967,7 +973,7 @@ def parser_flat(parser: argparse.ArgumentParser): "--fasta", required=False, type=Path, - help="A tab-separated file listing the genome names, and the fasta filepath of its genomic " + help="A tab-separated file listing genome names and the FASTA file paths of their genomic " "sequence(s) (the fastas can be compressed with gzip). One line per genome.", ) @@ -975,7 +981,7 @@ def parser_flat(parser: argparse.ArgumentParser): "--anno", required=False, type=Path, - help="A tab-separated file listing the genome names, and the gff/gbff filepath of its " + help="A tab-separated file listing genome names and the GFF/GBFF file paths of their annotations " "annotations (the files can be compressed with gzip). One line per genome. " "If this is provided, those annotations will be used.", ) diff --git a/ppanggolin/formats/writeFlatMetadata.py b/ppanggolin/formats/writeFlatMetadata.py index 8c4fae46..c8a0e36f 100644 --- a/ppanggolin/formats/writeFlatMetadata.py +++ b/ppanggolin/formats/writeFlatMetadata.py @@ -179,6 +179,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "write_metadata", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Writes 'TSV' files that represent the metadata associated with elements of the pangenome." + parser.category = "Output" parser_flat(parser) return parser @@ -203,14 +205,18 @@ def parser_flat(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) required.add_argument( "-o", "--output", required=True, type=Path, - help="Output directory where the file(s) will be written", + help="Output directory where the file(s) will be written.", ) optional = parser.add_argument_group(title="Optional arguments") diff --git a/ppanggolin/formats/writeFlatPangenome.py b/ppanggolin/formats/writeFlatPangenome.py index 0a680645..139d1afe 100644 --- a/ppanggolin/formats/writeFlatPangenome.py +++ b/ppanggolin/formats/writeFlatPangenome.py @@ -218,9 +218,9 @@ def write_gexf_header(gexf: TextIO, light: bool = True): if not light: index = pan.get_org_index() # has been computed already gexf.write( - '\n\n' - ) # TODO update link + '\n\n' + ) gexf.write(' \n') gexf.write(' \n') gexf.write(' \n') @@ -1686,6 +1686,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "write_pangenome", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Writes 'flat' files that represent the pangenome and its elements for use with other software." + parser.category = "Output" parser_flat(parser) return parser @@ -1701,14 +1703,18 @@ def parser_flat(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) required.add_argument( "-o", "--output", required=True, type=Path, - help="Output directory where the file(s) will be written", + help="Output directory where the file(s) will be written.", ) optional = parser.add_argument_group(title="Optional arguments") @@ -1849,7 +1855,7 @@ def parser_flat(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) diff --git a/ppanggolin/formats/writeMSA.py b/ppanggolin/formats/writeMSA.py index fea0a5b1..3338e349 100644 --- a/ppanggolin/formats/writeMSA.py +++ b/ppanggolin/formats/writeMSA.py @@ -462,6 +462,10 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser :return : parser arguments for align command """ parser = sub_parser.add_parser("msa", formatter_class=argparse.RawTextHelpFormatter) + parser.description = ( + "Compute Multiple Sequence Alignments for pangenome gene families." + ) + parser.category = "Analysis using reference pangenomes" parser_msa(parser) return parser @@ -476,14 +480,18 @@ def parser_msa(parser: argparse.ArgumentParser): title="Required arguments", description="The following arguments are required :" ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) required.add_argument( "-o", "--output", required=True, type=Path, - help="Output directory where the file(s) will be written", + help="Output directory where the file(s) will be written.", ) optional = parser.add_argument_group( @@ -504,7 +512,7 @@ def parser_msa(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.05, - help="minimum ratio of genomes in which the family must have multiple genes " + help="Minimum ratio of genomes in which the family must have multiple genes " "for it to be considered 'duplicated'", ) optional.add_argument( @@ -565,14 +573,14 @@ def parser_msa(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) optional.add_argument( "--tmpdir", required=False, type=str, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) diff --git a/ppanggolin/formats/writeSequences.py b/ppanggolin/formats/writeSequences.py index 81f1dfd3..316c5897 100644 --- a/ppanggolin/formats/writeSequences.py +++ b/ppanggolin/formats/writeSequences.py @@ -599,6 +599,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "fasta", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Writes fasta files for different elements of the pangenome." + parser.category = "Output" parser_seq(parser) return parser @@ -633,14 +635,18 @@ def parser_seq(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) required.add_argument( "-o", "--output", required=True, type=Path, - help="Output directory where the file(s) will be written", + help="Output directory where the file(s) will be written.", ) context = parser.add_argument_group( @@ -651,14 +657,14 @@ def parser_seq(parser: argparse.ArgumentParser): "--fasta", required=False, type=Path, - help="A tab-separated file listing the genome names, and the fasta filepath of its genomic " + help="A tab-separated file listing genome names and the FASTA file paths of their genomic " "sequence(s) (the fastas can be compressed with gzip). One line per genome.", ) context.add_argument( "--anno", required=False, type=Path, - help="A tab-separated file listing the genome names, and the gff/gbff filepath of its " + help="A tab-separated file listing genome names and the GFF/GBFF file paths of their annotations " "annotations (the files can be compressed with gzip). One line per genome. " "If this is provided, those annotations will be used.", ) @@ -728,14 +734,18 @@ def parser_seq(parser: argparse.ArgumentParser): "This can be accessed using 'ppanggolin info'.", ) optional.add_argument( - "--cpu", required=False, default=1, type=int, help="Number of available threads" + "--cpu", + required=False, + default=1, + type=int, + help="Number of available threads.", ) optional.add_argument( "--tmpdir", required=False, type=Path, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) optional.add_argument( "--keep_tmp", diff --git a/ppanggolin/graph/makeGraph.py b/ppanggolin/graph/makeGraph.py index ae877013..d6c1bfd7 100644 --- a/ppanggolin/graph/makeGraph.py +++ b/ppanggolin/graph/makeGraph.py @@ -172,6 +172,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "graph", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Create the pangenome graph" + parser.category = "Expert" parser_graph(parser) return parser @@ -186,7 +188,11 @@ def parser_graph(parser: argparse.ArgumentParser): title="Required arguments", description="Following arguments is required:" ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) optional = parser.add_argument_group(title="Optional arguments") optional.add_argument( diff --git a/ppanggolin/info/info.py b/ppanggolin/info/info.py index d6563bf0..9b0c3740 100644 --- a/ppanggolin/info/info.py +++ b/ppanggolin/info/info.py @@ -91,7 +91,7 @@ def print_info( if not (status or content or parameters or metadata): status, content, parameters, metadata = (True, True, True, True) - h5f = tables.open_file(pangenome, "r+") + h5f = tables.open_file(pangenome, "r") if status: print_yaml(read_status(h5f)) if content: @@ -125,6 +125,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "info", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Prints information about a given pangenome graph file." + parser.category = "Output" parser_info(parser) return parser @@ -144,7 +146,7 @@ def parser_info(parser: argparse.ArgumentParser): "--pangenome", required=True, type=Path, - help="Path to the pangenome .h5 file", + help="Path to the pangenome .h5 file.", ) options = parser.add_argument_group( diff --git a/ppanggolin/main.py b/ppanggolin/main.py index fbf85c3c..8a831a28 100644 --- a/ppanggolin/main.py +++ b/ppanggolin/main.py @@ -1,112 +1,99 @@ -#!/usr/bin/env python3 - # default libraries -import sys - -if sys.version_info < (3, 9): # minimum is python3.9 - raise AssertionError( - "Minimum python version to run PPanGGOLiN is 3.9. Your current python version is " - + ".".join(map(str, sys.version_info)) - ) import argparse +import sys -# local modules -import ppanggolin.pangenome -from ppanggolin.utils import ( - check_input_files, - set_verbosity_level, - add_common_arguments, - manage_cli_and_config_args, -) -import ppanggolin.nem.partition -import ppanggolin.nem.rarefaction -import ppanggolin.graph +import ppanggolin.align import ppanggolin.annotate import ppanggolin.cluster +import ppanggolin.context import ppanggolin.figures import ppanggolin.formats +import ppanggolin.graph import ppanggolin.info +import ppanggolin.meta import ppanggolin.metrics -import ppanggolin.align -import ppanggolin.RGP import ppanggolin.mod -import ppanggolin.context -import ppanggolin.workflow -import ppanggolin.meta -import ppanggolin.utility +import ppanggolin.nem.partition +import ppanggolin.nem.rarefaction +# local modules +import ppanggolin.RGP +import ppanggolin.utility +import ppanggolin.workflow from ppanggolin import ( SUBCOMMAND_TO_SUBPARSER, epilog, + mod_epilog, pan_epilog, rgp_epilog, - mod_epilog, version, ) +from ppanggolin.utils import ( + RawTextHelpFormatterWithDefaults, + add_common_arguments, + check_input_files, + manage_cli_and_config_args, + set_verbosity_level, +) -def cmd_line() -> argparse.Namespace: - """Manage the command line argument given by user +def _build_command_summary(parser: argparse.ArgumentParser) -> str: + """Build the human-readable command list from each subparser's own metadata.""" + category_order = [ + "Basic", + "Expert", + "Output", + "Regions of Genomic Plasticity", + "Analysis using reference pangenomes", + "Utility command", + ] - :return: arguments given and readable by PPanGGOLiN - """ - # need to manually write the description so that it's displayed into groups of subcommands .... - desc = "\n" - desc += ( - "All of the following subcommands have their own set of options. To see them for a given subcommand," - " use it with -h or --help, as such:\n" - ) - desc += " ppanggolin -h\n" - desc += "\n" - desc += " Basic:\n" - desc += " all Easy workflow to run all possible analysis\n" - desc += " workflow Easy workflow to run a pangenome analysis in one go\n" - desc += ( - " panrgp Easy workflow to run a pangenome analysis with genomic islands and spots of" - " insertion detection\n" - ) - desc += " panmodule Easy workflow to run a pangenome analysis with module prediction\n" - desc += " \n" - desc += " Expert:\n" - desc += " annotate Annotate genomes\n" - desc += " cluster Cluster genes into gene families\n" - desc += " graph Create the pangenome graph\n" - desc += " partition Partition the pangenome graph\n" - desc += " rarefaction Compute the rarefaction curve of the pangenome\n" - desc += " metadata Add metadata to elements in yout pangenome\n" - desc += " \n" - desc += " Output:\n" - desc += " draw Draw figures representing the pangenome through different aspects\n" - desc += " write_pangenome Writes 'flat' files that represent the pangenome and its elements for use with other software.\n" - desc += " write_genomes Writes 'flat' files that represent the genomes along with their associated pangenome elements.\n" - desc += " write_metadata Writes 'TSV' files that represent the metadata associated with elements of the pangenome.\n" - desc += " fasta Writes fasta files for different elements of the pangenome.\n" - desc += ( - " info Prints information about a given pangenome graph file.\n" + subparser_action = next( + ( + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ), + None, ) - desc += " metrics Compute several metrics on a given pangenome.\n" - desc += " \n" - desc += " Regions of Genomic Plasticity:\n" - desc += " rgp Predicts Regions of Genomic Plasticity in the genomes of your pangenome.\n" - desc += " spot Predicts spots in your pangenome.\n" - desc += " module Predicts functional modules in your pangenome.\n" - desc += " rgp_cluster Cluster RGPs based on their gene families.\n" - desc += " \n" - desc += " Analysis using reference pangenomes:\n" - desc += " msa Compute Multiple Sequence Alignments for pangenome gene families.\n" - desc += ( - " align Aligns a genome or a set of proteins to the pangenome gene families and " - "predicts information from it.\n" - ) - desc += " context Local genomic context analysis.\n" - desc += " projection Annotates external genomes with an existing pangenome.\n" - desc += " \n" - desc += " Utility command:\n" - desc += " utils Helper side commands." + if subparser_action is None: + return "" + + grouped = {category: [] for category in category_order} + for command_name, command_parser in subparser_action.choices.items(): + category = getattr(command_parser, "category", "General") + description = (getattr(command_parser, "description", "") or "").strip() + grouped.setdefault(category, []).append((command_name, description)) + + lines = [ + "All of the following subcommands have their own set of options. To see them for a given subcommand,", + " use it with -h or --help, as such:", + " ppanggolin -h", + "", + ] + + for category in category_order: + if category not in grouped or not grouped[category]: + continue + lines.append(f" {category}:") + for command_name, description in sorted( + grouped[category], key=lambda item: item[0] + ): + lines.append(f" {command_name:<15} {description}") + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + +def build_parser() -> argparse.ArgumentParser: + """Build the PPanGGOLiN command-line parser without parsing user input.""" + + argparse.RawTextHelpFormatter = RawTextHelpFormatterWithDefaults parser = argparse.ArgumentParser( + prog="ppanggolin", description="Depicting microbial species diversity via a Partitioned PanGenome Graph Of Linked Neighbors", - formatter_class=argparse.RawTextHelpFormatter, + formatter_class=RawTextHelpFormatterWithDefaults, epilog=epilog + pan_epilog + rgp_epilog + mod_epilog, ) @@ -115,20 +102,12 @@ def cmd_line() -> argparse.Namespace: ) subparsers = parser.add_subparsers( - metavar="", dest="subcommand", title="subcommands", description=desc + dest="subcommand", title="subcommands", metavar="" ) subparsers.required = True # because python3 sent subcommands to hell apparently - # print help if no subcommand is specified - if len(sys.argv) == 1: - parser.print_help() - sys.exit(0) - - # manage command parser to use command arguments - subs = [] for sub_cmd, sub_fct in SUBCOMMAND_TO_SUBPARSER.items(): sub = sub_fct(subparsers) - # add options common to all subcommands add_common_arguments(sub) sub.epilog = epilog if sub_cmd not in ["rgp", "spot", "module", "rgp_cluster"]: @@ -145,16 +124,30 @@ def cmd_line() -> argparse.Namespace: sub.epilog += rgp_epilog if sub_cmd not in ["rgp", "spot", "rgp_cluster", "panrgp"]: sub.epilog += mod_epilog - subs.append(sub) - # manage command without common arguments - sub_info = ppanggolin.info.subparser(subparsers) - sub_utils = ppanggolin.utility.utils.subparser(subparsers) - subs += [sub_info, sub_utils] + # if getattr(sub, "description", None): + # sub.help = sub.description + + parser.epilog = _build_command_summary(parser) + parser.epilog + return parser + + +def cmd_line() -> argparse.Namespace: + """Manage the command line argument given by user + + :return: arguments given and readable by PPanGGOLiN + """ + parser = build_parser() + + # print help if no subcommand is specified + if len(sys.argv) == 1: + parser.print_help() + sys.exit(0) # print help if only the command is given - for sub in subs: - if len(sys.argv) == 2 and sub.prog.split()[1] == sys.argv[1]: + subcommands = parser._subparsers._group_actions[0].choices + for sub_name, sub in subcommands.items(): + if len(sys.argv) == 2 and sub_name == sys.argv[1]: sub.print_help() exit(0) diff --git a/ppanggolin/meta/meta.py b/ppanggolin/meta/meta.py index 7626d21a..d02aa22e 100644 --- a/ppanggolin/meta/meta.py +++ b/ppanggolin/meta/meta.py @@ -262,6 +262,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "metadata", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Add metadata to elements in yout pangenome" + parser.category = "Expert" parser_meta(parser) return parser @@ -277,7 +279,11 @@ def parser_meta(parser: argparse.ArgumentParser): description="All of the following arguments are required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) required.add_argument( "-m", @@ -315,7 +321,7 @@ def parser_meta(parser: argparse.ArgumentParser): if __name__ == "__main__": """To test local change and allow using debugger""" - from ppanggolin.utils import check_log, set_verbosity_level + from ppanggolin.utils import add_common_arguments, set_verbosity_level main_parser = argparse.ArgumentParser( description="Depicting microbial species diversity via a Partitioned PanGenome Graph Of Linked Neighbors", @@ -323,42 +329,14 @@ def parser_meta(parser: argparse.ArgumentParser): ) parser_meta(main_parser) - common = main_parser.add_argument_group(title="Common argument") - common.add_argument( - "--verbose", - required=False, - type=int, - default=1, - choices=[0, 1, 2], - help="Indicate verbose level (0 for warning and errors only, 1 for info, 2 for debug)", - ) - common.add_argument( - "--log", - required=False, - type=check_log, - default="stdout", - help="log output file", - ) - common.add_argument( - "-d", - "--disable_prog_bar", - required=False, - action="store_true", - help="disables the progress bars", - ) - common.add_argument( + main_parser.add_argument( "-c", "--cpu", required=False, default=1, type=int, - help="Number of available cpus", - ) - common.add_argument( - "-f", - "--force", - action="store_true", - help="Force writing in output directory and in pangenome output file.", + help="Number of available CPUs.", ) + add_common_arguments(main_parser) set_verbosity_level(main_parser.parse_args()) launch(main_parser.parse_args()) diff --git a/ppanggolin/metadata.py b/ppanggolin/metadata.py index c78f916f..d6a1edc6 100644 --- a/ppanggolin/metadata.py +++ b/ppanggolin/metadata.py @@ -44,6 +44,8 @@ def __init__(self, source: str, **kwargs): for attr, value in kwargs.items(): if isinstance(value, list): value = self._join_list(value) + if isinstance(value, str): # for gb_io + value = value.replace("\n", " ") if value is not None and not isna(value): setattr(self, attr, value) diff --git a/ppanggolin/metrics/metrics.py b/ppanggolin/metrics/metrics.py index 6dce8c5f..034ead72 100644 --- a/ppanggolin/metrics/metrics.py +++ b/ppanggolin/metrics/metrics.py @@ -28,7 +28,7 @@ def check_already_computed_metric( :param print_metric: Print metrics if already computed :param recompute: Are metrics going to be recompute """ - with tables.open_file(pangenome.file, "a") as h5f: + with tables.open_file(pangenome.file, "r") as h5f: info_group = h5f.root.info if genomes_fluidity and "genomes_fluidity" in info_group._v_attrs._f_list(): logging.getLogger("PPanGGOLiN").warning( @@ -154,6 +154,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "metrics", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Compute several metrics on a given pangenome." + parser.category = "Output" parser_metrics(parser) return parser @@ -172,7 +174,7 @@ def parser_metrics(parser: argparse.ArgumentParser): "--pangenome", required=False, type=Path, - help="Path to the pangenome .h5 file", + help="Path to the pangenome .h5 file.", ) onereq = parser.add_argument_group( diff --git a/ppanggolin/mod/module.py b/ppanggolin/mod/module.py index 81dd900e..05f939df 100644 --- a/ppanggolin/mod/module.py +++ b/ppanggolin/mod/module.py @@ -206,6 +206,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "module", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Predicts functional modules in your pangenome." + parser.category = "Regions of Genomic Plasticity" parser_module(parser) return parser @@ -221,7 +223,11 @@ def parser_module(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) optional = parser.add_argument_group(title="Optional arguments") optional.add_argument( @@ -264,7 +270,7 @@ def parser_module(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.85, - help="minimum jaccard similarity used to filter edges between gene families. " + help="Minimum Jaccard similarity used to filter edges between gene families. " "Increasing it will improve precision but lower sensitivity a lot.", ) @@ -274,7 +280,7 @@ def parser_module(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) diff --git a/ppanggolin/nem/partition.py b/ppanggolin/nem/partition.py index 60c4192e..c7699e57 100644 --- a/ppanggolin/nem/partition.py +++ b/ppanggolin/nem/partition.py @@ -341,6 +341,15 @@ def nem_samples( return partition_nem(*pack) +def ordered_organisms(organisms) -> list: + """ + Genomes in a stable, process-independent order. + :param organisms: genomes to order + :return: the genomes sorted by name + """ + return sorted(organisms, key=lambda org: org.name) + + def write_nem_input_files( tmpdir: Path, organisms: set, sm_degree: int = 10 ) -> Tuple[float, int]: @@ -355,9 +364,10 @@ def write_nem_input_files( """ mk_outdir(tmpdir, force=False) total_edges_weight = 0 + columns = ordered_organisms(organisms) with open(tmpdir / "column_org_file", "w") as org_file: - org_file.write(" ".join([f'"{org.name}"' for org in organisms]) + "\n") + org_file.write(" ".join([f'"{org.name}"' for org in columns]) + "\n") logging.getLogger("PPanGGOLiN").debug( "Writing nem_file.str nem_file.index nem_file.nei and nem_file.dat files" @@ -374,7 +384,7 @@ def write_nem_input_files( index_org = {} default_dat = [] - for index, org in enumerate(organisms): + for index, org in enumerate(columns): default_dat.append("0") index_org[org] = index @@ -471,8 +481,9 @@ def evaluate_nb_partitions( tmpdir = Path(tempfile.gettempdir()) if tmpdir is None else tmpdir newtmpdir = tmpdir / "eval_partitions" + random.seed(seed) # Seeded before forking if len(organisms) > chunk_size: - select_organisms = set(random.sample(list(organisms), chunk_size)) + select_organisms = set(random.sample(ordered_organisms(organisms), chunk_size)) else: select_organisms = set(organisms) @@ -533,7 +544,7 @@ def evaluate_nb_partitions( best_k = chosen_k if len(all_bics) > 3: - max_icl_k = max(all_icls, key=all_icls.get) + max_icl_k = max(sorted(all_icls), key=all_icls.get) delta_icl = (all_icls[max_icl_k] - min(all_icls.values())) * icl_margin best_k = min( { @@ -809,8 +820,8 @@ def validate_family(res): prev = len(samples) # if we've been sampling already, samples is not empty. while not all(val >= condition for val in org_nb_sample.values()): # each family must be tested at least len(select_organisms)/chunk_size times. - shuffled_orgs = list(organisms) # copy select_organisms - random.shuffle(shuffled_orgs) # shuffle the copied list + shuffled_orgs = ordered_organisms(organisms) + random.shuffle(shuffled_orgs) # seeded shuffle while len(shuffled_orgs) > chunk_size: samples.append(set(shuffled_orgs[:chunk_size])) for org in samples[-1]: @@ -943,6 +954,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "partition", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Partition the pangenome graph" + parser.category = "Expert" parser_partition(parser) return parser @@ -958,7 +971,11 @@ def parser_partition(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome.h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) optional = parser.add_argument_group(title="Optional arguments") @@ -988,7 +1005,7 @@ def parser_partition(parser: argparse.ArgumentParser): f"ppanggolin_output{time.strftime('DATE%Y-%m-%d_HOUR%H.%M.%S', time.localtime())}" f"_PID{str(os.getpid())}" ), - help="Output directory", + help="Output directory.", ) optional.add_argument( "-fd", @@ -1059,7 +1076,7 @@ def parser_partition(parser: argparse.ArgumentParser): "--seed", type=int, default=42, - help="seed used to generate random numbers", + help="Seed used to generate random numbers.", ) optional.add_argument( "-c", @@ -1067,14 +1084,14 @@ def parser_partition(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) optional.add_argument( "--tmpdir", required=False, type=str, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) diff --git a/ppanggolin/nem/rarefaction.py b/ppanggolin/nem/rarefaction.py index 94fcb1e6..47b3320f 100644 --- a/ppanggolin/nem/rarefaction.py +++ b/ppanggolin/nem/rarefaction.py @@ -136,8 +136,8 @@ def validate_family( while not all(val >= condition for val in org_nb_sample.values()): # each family must be tested at least len(select_organisms)/chunk_size times. - shuffled_orgs = list(samp) # copy select_organisms - random.shuffle(shuffled_orgs) # shuffle the copied list + shuffled_orgs = ppp.ordered_organisms(samp) + random.shuffle(shuffled_orgs) # seeded shuffle while len(shuffled_orgs) > chunk_size: org_samples.append(set(shuffled_orgs[:chunk_size])) for org in org_samples[-1]: @@ -767,6 +767,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser description="Compute the rarefaction curve of the pangenome", formatter_class=argparse.RawTextHelpFormatter, ) + parser.description = "Compute the rarefaction curve of the pangenome" + parser.category = "Expert" parser_rarefaction(parser) return parser @@ -782,7 +784,11 @@ def parser_rarefaction(parser: argparse.ArgumentParser): description="One of the following arguments is required :", ) required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome .h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) optional = parser.add_argument_group(title="Optional arguments") @@ -835,7 +841,7 @@ def parser_rarefaction(parser: argparse.ArgumentParser): f"ppanggolin_output{time.strftime('DATE%Y-%m-%d_HOUR%H.%M.%S', time.localtime())}" f"_PID{str(os.getpid())}" ), - help="Output directory", + help="Output directory.", ) optional.add_argument( "-fd", @@ -903,14 +909,14 @@ def parser_rarefaction(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) optional.add_argument( "--tmpdir", required=False, type=str, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) diff --git a/ppanggolin/projection/projection.py b/ppanggolin/projection/projection.py index 91e77db1..9f5b6824 100644 --- a/ppanggolin/projection/projection.py +++ b/ppanggolin/projection/projection.py @@ -1711,6 +1711,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "projection", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Annotates external genomes with an existing pangenome." + parser.category = "Analysis using reference pangenomes" parser_projection(parser) return parser @@ -1724,7 +1726,11 @@ def parser_projection(parser: argparse.ArgumentParser): required = parser.add_argument_group(title="Required arguments") required.add_argument( - "-p", "--pangenome", required=False, type=Path, help="The pangenome.h5 file" + "-p", + "--pangenome", + required=False, + type=Path, + help="Path to the pangenome .h5 file.", ) required.add_argument( @@ -1732,7 +1738,8 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, type=Path, help="Specify a FASTA file containing the genomic sequences of the genome(s) you wish to annotate, " - "or provide a tab-separated file listing genome names alongside their respective FASTA filepaths, with one line per genome.", + "or provide a tab-separated file listing genome names alongside their respective FASTA file paths, " + "with one line per genome.", ) required.add_argument( @@ -1740,7 +1747,7 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, type=Path, help="Specify an annotation file in GFF/GBFF format for the genome you wish to annotate. " - "Alternatively, you can provide a tab-separated file listing genome names alongside their respective annotation filepaths, " + "Alternatively, you can provide a tab-separated file listing genome names alongside their respective annotation file paths, " "with one line per genome. If both an annotation file and a FASTA file are provided, the annotation file will take precedence.", ) @@ -1755,7 +1762,7 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, type=str, default="input_genome", - help="Specify the name of the genome whose genome you want to annotate when providing a single FASTA or annotation file.", + help="Specify the name of the genome to annotate when providing a single FASTA or annotation file.", ) required_single.add_argument( @@ -1777,7 +1784,7 @@ def parser_projection(parser: argparse.ArgumentParser): + time.strftime("_DATE%Y-%m-%d_HOUR%H.%M.%S", time.localtime()) + "_PID" + str(os.getpid()), - help="Output directory", + help="Output directory.", ) optional.add_argument( @@ -1794,8 +1801,7 @@ def parser_projection(parser: argparse.ArgumentParser): "--no_defrag", required=False, action="store_true", - help="DO NOT Realign gene families to link fragments with " - "their non-fragmented gene family. (default: False)", + help="Do not realign gene families to link fragments with their non-fragmented gene family. (Default: False)", ) optional.add_argument( @@ -1811,7 +1817,7 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.8, - help="min identity percentage threshold", + help="Minimum identity percentage threshold.", ) optional.add_argument( @@ -1819,7 +1825,7 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.8, - help="min coverage percentage threshold", + help="Minimum coverage percentage threshold.", ) optional.add_argument( @@ -1827,7 +1833,7 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, action="store_true", help="In the context of provided annotation, use this option to read pseudogenes. " - "(Default behavior is to ignore them)", + "(Default behavior is to ignore them).", ) optional.add_argument( @@ -1835,9 +1841,8 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.05, - help="minimum ratio of genomes in which the family must have multiple genes " - "for it to be considered 'duplicated'. " - "This metric is used to compute completeness and duplication of the input genomes", + help="Minimum ratio of genomes in which the family must have multiple genes for it to be considered 'duplicated'. " + "This metric is used to compute completeness and duplication of the input genomes.", ) optional.add_argument( @@ -1845,7 +1850,7 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.95, - help="Soft core threshold used when generating general statistics on the projected genome. " + help="Soft-core threshold used when generating general statistics on the projected genome. " "This threshold does not influence PPanGGOLiN's partitioning. " "The value determines the minimum fraction of genomes that must possess a gene family " "for it to be considered part of the soft core.", @@ -1855,8 +1860,8 @@ def parser_projection(parser: argparse.ArgumentParser): "--spot_graph", required=False, action="store_true", - help="Write the spot graph to a file, with pairs of blocks of single copy markers flanking RGPs " - "as nodes. This graph can be used to visualize nodes that have RGPs from the input genome.", + help="Write the spot graph to a file, with pairs of blocks of single-copy markers flanking RGPs as nodes. " + "This graph can be used to visualize nodes that have RGPs from the input genome.", ) optional.add_argument( @@ -1887,14 +1892,14 @@ def parser_projection(parser: argparse.ArgumentParser): "--table", required=False, action="store_true", - help="Generate a tsv file for each input genome with pangenome annotations.", + help="Generate a TSV file for each input genome with pangenome annotations.", ) optional.add_argument( "--compress", required=False, action="store_true", - help="Compress the files in .gz", + help="Compress the files in .gz.", ) optional.add_argument( @@ -1910,7 +1915,7 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) optional.add_argument( @@ -1918,7 +1923,7 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, type=Path, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) optional.add_argument( @@ -1926,7 +1931,7 @@ def parser_projection(parser: argparse.ArgumentParser): required=False, default=False, action="store_true", - help="Keeping temporary files (useful for debugging).", + help="Keep temporary files (useful for debugging).", ) optional.add_argument( diff --git a/ppanggolin/utility/utils.py b/ppanggolin/utility/utils.py index 0556ee59..9cb2c844 100644 --- a/ppanggolin/utility/utils.py +++ b/ppanggolin/utility/utils.py @@ -283,6 +283,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "utils", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Helper side commands." + parser.category = "Utility command" parser_default_config(parser) return parser @@ -320,38 +322,20 @@ def parser_default_config(parser: argparse.ArgumentParser): help="name and path of the config file with default parameters written in yaml.", ) - optional.add_argument( - "--verbose", - required=False, - type=int, - default=1, - choices=[0, 1, 2], - help="Indicate verbose level (0 for warning and errors only, 1 for info, 2 for debug)", - ) - - optional.add_argument( - "--log", - required=False, - type=check_log, - default="stdout", - help="log output file", - ) - - optional.add_argument( - "-f", - "--force", - action="store_true", - help="Overwrite the given output file if it exists.", - ) + # Common CLI options are added centrally in add_common_arguments() to keep + # a single source of truth and avoid argparse conflicts. if __name__ == "__main__": """To test local change and allow using debugger""" + from ppanggolin.utils import add_common_arguments + main_parser = argparse.ArgumentParser( description="Depicting microbial species diversity via a Partitioned PanGenome Graph Of Linked Neighbors", formatter_class=argparse.RawTextHelpFormatter, ) parser_default_config(main_parser) + add_common_arguments(main_parser) launch(main_parser.parse_args()) diff --git a/ppanggolin/utils.py b/ppanggolin/utils.py index 06ccb68a..cd511f7c 100755 --- a/ppanggolin/utils.py +++ b/ppanggolin/utils.py @@ -107,6 +107,12 @@ DRAW_FLAG_DEFAULT_IN_WF = ["tile_plot", "ucurve", "draw_spots"] +class RawTextHelpFormatterWithDefaults( + argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter +): + """Preserve multiline formatting while appending default values to help text.""" + + def check_log(log_file: str) -> TextIO: """ Check if the output log is writable @@ -154,44 +160,7 @@ def check_tsv_sanity(tsv_file: Path): :param tsv: Path to the TSV containing organism information. :raises ValueError: If the file format is incorrect or contains invalid genome names. """ - with read_compressed_or_not(tsv_file) as input_file: - name_set = set() - duplicated_names = set() - non_existing_files = set() - - for line in input_file: - elements = [el.strip() for el in line.split("\t")] - - if len(elements) <= 1: - raise ValueError(f"No tabulation separator found in file: {tsv_file}") - - genome_name, genome_path = elements[0], elements[1] - - if " " in genome_name: - raise ValueError( - f"Genome names cannot contain spaces (first encountered: '{genome_name}'). " - "Please remove spaces to ensure compatibility with PPanGGOLiN dependencies." - ) - - if genome_name in name_set: - duplicated_names.add(genome_name) - name_set.add(genome_name) - - org_path = Path(genome_path) - if ( - not org_path.exists() - and not tsv_file.parent.joinpath(org_path).exists() - ): - non_existing_files.add(genome_path) - - if non_existing_files: - raise ValueError( - f"Some specified genome files do not exist: {', '.join(non_existing_files)}" - ) - if duplicated_names: - raise ValueError( - f"Some genome names are duplicated: {', '.join(duplicated_names)}" - ) + parse_input_paths_file(tsv_file) def check_input_files(file: Path, check_tsv: bool = False): @@ -587,7 +556,7 @@ def check_option_workflow(args): ) -def parse_config_file(yaml_config_file: str) -> dict: +def parse_config_file(yaml_config_file: Path) -> dict: """ Parse yaml config file. @@ -596,7 +565,7 @@ def parse_config_file(yaml_config_file: str) -> dict: :return: dict of config with key the command and as value another dict with param as key and value as value. """ - with yaml_config_file as yaml_fh: + with open(yaml_config_file) as yaml_fh: config = yaml.safe_load(yaml_fh) if config is None: @@ -623,44 +592,82 @@ def add_common_arguments(subparser: argparse.ArgumentParser): :param subparser: A subparser object from any subcommand. """ + existing_options = { + option for action in subparser._actions for option in action.option_strings + } + common = subparser._action_groups.pop( 1 ) # get the 'optional arguments' action group. common.title = "Common arguments" - common.add_argument( - "--verbose", - required=False, - type=int, - default=1, - choices=[0, 1, 2], - help="Indicate verbose level (0 for warning and errors only, 1 for info, 2 for debug)", - ) - common.add_argument( - "--log", - required=False, - type=check_log, - default="stdout", - help="log output file", - ) - common.add_argument( - "-d", - "--disable_prog_bar", - required=False, - action="store_true", - help="disables the progress bars", - ) - common.add_argument( - "-f", - "--force", - action="store_true", - help="Force writing in output directory and in pangenome output file.", - ) - common.add_argument( - "--config", - required=False, - type=argparse.FileType(), - help="Specify command arguments through a YAML configuration file.", - ) + + if "--verbose" not in existing_options: + common.add_argument( + "--verbose", + required=False, + type=int, + default=1, + choices=[0, 1, 2], + help="Verbose level: 0 = warnings and errors only, 1 = info, 2 = debug.", + ) + else: + logging.getLogger("PPanGGOLiN").warning( + "The --verbose argument is already defined in the subcommand. " + "Please remove it from the subcommand to avoid conflicts." + ) + + if "--log" not in existing_options: + common.add_argument( + "--log", + required=False, + type=check_log, + default="stdout", + help="Log output file.", + ) + else: + logging.getLogger("PPanGGOLiN").warning( + "The --log argument is already defined in the subcommand. " + "Please remove it from the subcommand to avoid conflicts." + ) + + if "-d" not in existing_options and "--disable_prog_bar" not in existing_options: + common.add_argument( + "-d", + "--disable_prog_bar", + required=False, + action="store_true", + help="Disable the progress bars.", + ) + else: + logging.getLogger("PPanGGOLiN").warning( + "The --disable_prog_bar argument is already defined in the subcommand. " + "Please remove it from the subcommand to avoid conflicts." + ) + + if "-f" not in existing_options and "--force" not in existing_options: + common.add_argument( + "-f", + "--force", + action="store_true", + help="Force overwrite in the output directory and pangenome output file.", + ) + else: + logging.getLogger("PPanGGOLiN").warning( + "The --force argument is already defined in the subcommand. " + "Please remove it from the subcommand to avoid conflicts." + ) + if "--config" not in existing_options: + common.add_argument( + "--config", + required=False, + type=Path, + help="Specify command arguments through a YAML configuration file.", + ) + else: + logging.getLogger("PPanGGOLiN").warning( + "The --config argument is already defined in the subcommand. " + "Please remove it from the subcommand to avoid conflicts." + ) subparser._action_groups.append(common) @@ -808,7 +815,7 @@ def get_args_differing_from_default( def manage_cli_and_config_args( - subcommand: str, config_file: str, subcommand_to_subparser: dict + subcommand: str, config_file: Optional[Path], subcommand_to_subparser: dict ) -> argparse.Namespace: """ Manage command line and config arguments for the given subcommand. @@ -1326,41 +1333,90 @@ def parse_input_paths_file( This function reads an input paths file, which is in TSV format, and extracts genome information including file paths and putative circular contigs. + Each non-commented line must have at least two tab-separated columns: + genome_namegenome_file_path[optional circular contig identifiers]. + :param path_list_file: The path to the input paths file. - :return: A dictionary where keys are genome names and values are dictionaries containing path information and - putative circular contigs. - :raises FileNotFoundError: If a specified genome file path does not exist. - :raises Exception: If there are no genomes in the provided file. + :return: A dictionary mapping each genome name to its path and circular contigs. + :raises ValueError: If the TSV is malformed, has missing fields, duplicate names, + blank names or whitespace in names. + :raises FileNotFoundError: If a genome file path does not exist. + :raises ValueError: If the file contains no valid genomes. """ logging.getLogger("PPanGGOLiN").info( f"Reading {path_list_file} to process genome files" ) genome_name_to_genome_path = {} - for line in read_compressed_or_not(path_list_file): - elements = [el.strip() for el in line.split("\t")] - genome_file_path = Path(elements[1]) - genome_name = elements[0] - putative_circular_contigs = elements[2:] + with read_compressed_or_not(path_list_file) as input_file: + for line_number, line in enumerate(input_file, start=1): + stripped_line = line.strip() + if not stripped_line or stripped_line.startswith("#"): + continue + + elements = [el.strip() for el in line.rstrip("\n").split("\t")] + if len(elements) < 2: + raise ValueError( + f"Malformed genome list line {line_number} in '{path_list_file}': " + "expected at least 2 tab-separated columns (genome name and genome file path), " + f"but found {len(elements)} column(s): '{line.rstrip()}'." + ) - if not genome_file_path.exists(): - # Check if the file path doesn't exist and try an alternative path. - genome_file_path_alt = path_list_file.parent.joinpath(genome_file_path) + genome_name = elements[0] + genome_path = elements[1] - if not genome_file_path_alt.exists(): - raise FileNotFoundError( - f"The file path '{genome_file_path}' for genome '{genome_name}' specified in '{path_list_file}' does not exist." + if not genome_name: + raise ValueError( + f"Malformed genome list line {line_number} in '{path_list_file}': " + "the genome name is empty." ) - else: + + if any(ch.isspace() for ch in genome_name): + raise ValueError( + f"Genome name '{genome_name}' on line {line_number} in '{path_list_file}' " + "must not contain whitespace. Please use names without spaces." + ) + + trimmed_genome_name = genome_name.strip() + if trimmed_genome_name != genome_name: + genome_name = trimmed_genome_name + + if genome_name in genome_name_to_genome_path: + raise ValueError( + f"Duplicate genome name '{genome_name}' found in '{path_list_file}' on line {line_number} " + "after trimming whitespace. Genome names must be unique." + ) + + if not genome_path: + raise ValueError( + f"Malformed genome list line {line_number} in '{path_list_file}': " + f"the genome file path for genome '{genome_name}' is empty." + ) + + genome_file_path = Path(genome_path) + if not genome_file_path.exists(): + genome_file_path_alt = path_list_file.parent.joinpath(genome_file_path) + if not genome_file_path_alt.exists(): + raise FileNotFoundError( + f"The file path '{genome_path}' for genome '{genome_name}' specified in '{path_list_file}' does not exist." + ) genome_file_path = genome_file_path_alt - genome_name_to_genome_path[genome_name] = { - "path": genome_file_path, - "circular_contigs": putative_circular_contigs, - } + if genome_file_path.is_dir(): + raise ValueError( + f"The genome file path '{genome_path}' for genome '{genome_name}' specified in '{path_list_file}' " + "resolves to a directory, not a file." + ) + + genome_name_to_genome_path[genome_name] = { + "path": genome_file_path, + "circular_contigs": elements[2:], + } if len(genome_name_to_genome_path) == 0: - raise Exception(f"There are no genomes in the provided file: {path_list_file} ") + raise ValueError( + f"There are no valid genomes in the provided file: {path_list_file}" + ) return genome_name_to_genome_path @@ -1767,3 +1823,29 @@ def check_translation_table_to_use( f"Using the default translation table: {user_translation_table}." ) return int(user_translation_table) + + +class Timer: + def __init__(self, name: str = None, logger: logging.Logger = None): + """ + Context manager to measure execution time. + + :param name: Optional name of the code block + :param logger: Optional logger to log the elapsed time + """ + self.name = name + self.logger = logger + self.start_time = None + self.elapsed = None + + def __enter__(self): + self.start_time = time.perf_counter() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.elapsed = time.perf_counter() - self.start_time + msg = f"[Timer:{self.name if self.name else ''}] {self.elapsed:.4f} seconds" + if self.logger: + self.logger.info(msg) + else: + print(msg) diff --git a/ppanggolin/workflow/all.py b/ppanggolin/workflow/all.py index 0f6caf04..330131c1 100644 --- a/ppanggolin/workflow/all.py +++ b/ppanggolin/workflow/all.py @@ -485,6 +485,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser :return : parser arguments for all command """ parser = sub_parser.add_parser("all", formatter_class=argparse.RawTextHelpFormatter) + parser.description = "Easy workflow to run all possible analysis" + parser.category = "Basic" add_workflow_args(parser) @@ -552,27 +554,24 @@ def add_workflow_args(parser: argparse.ArgumentParser): "--fasta", required=False, type=Path, - help="A tab-separated file listing the genome names, " - "and the fasta filepath of its genomic sequence(s) (the fastas can be compressed). " - "One line per genome. This option can be used alone.", + help="A tab-separated file listing genome names and the FASTA file paths of their genomic sequence(s) " + "(the FASTA files can be compressed). One line per genome. This option can be used alone.", ) required.add_argument( "--anno", required=False, type=Path, - help="A tab-separated file listing the genome names, and the gff filepath of " - "its annotations (the gffs can be compressed). One line per genome. " - "This option can be used alone IF the fasta sequences are in the gff files, " - "otherwise --fasta needs to be used.", + help="A tab-separated file listing genome names and the GFF file paths of their annotations " + "(the GFF files can be compressed). One line per genome. This option can be used alone if the FASTA " + "sequences are already present in the GFF files; otherwise, --fasta must be used.", ) required.add_argument( "--clusters", required=False, type=Path, - help="a tab-separated file listing the cluster names, the gene IDs, " - "and optionally whether they are a fragment or not.", + help="A tab-separated file listing cluster names, gene IDs, and optionally whether they are fragments.", ) optional = parser.add_argument_group(title="Optional arguments") @@ -583,14 +582,14 @@ def add_workflow_args(parser: argparse.ArgumentParser): required=False, type=Path, default=Path(f"ppanggolin_output{date}_PID{str(os.getpid())}"), - help="Output directory", + help="Output directory.", ) optional.add_argument( "--basename", required=False, default="pangenome", - help="basename for the output file", + help="Basename for the output file.", ) optional.add_argument( @@ -598,7 +597,7 @@ def add_workflow_args(parser: argparse.ArgumentParser): required=False, action="store_true", dest="rarefaction_flag", - help="Use to compute the rarefaction curves (WARNING: can be time consuming)", + help="Compute the rarefaction curves. Warning: this can be time consuming.", ) optional.add_argument( @@ -607,7 +606,7 @@ def add_workflow_args(parser: argparse.ArgumentParser): required=False, default=1, type=int, - help="Number of available cpus", + help="Number of available CPUs.", ) optional.add_argument( @@ -627,8 +626,7 @@ def add_workflow_args(parser: argparse.ArgumentParser): type=str.lower, default="bacteria", choices=["bacteria", "archaea"], - help="Kingdom to which the prokaryota belongs to, " - "to know which models to use for rRNA annotation.", + help="Kingdom to which the prokaryote belongs, to determine which models to use for rRNA annotation.", ) optional.add_argument( @@ -636,8 +634,8 @@ def add_workflow_args(parser: argparse.ArgumentParser): required=False, default="1", choices=["0", "1", "2", "3"], - help="the cluster mode of MMseqs2. 0: Setcover, 1: single linkage (or connected component)," - " 2: CD-HIT-like, 3: CD-HIT-like (lowmem)", + help="MMseqs2 clustering mode: 0 = Setcover, 1 = single linkage (or connected component), " + "2 = CD-HIT-like, 3 = CD-HIT-like (lowmem).", ) optional.add_argument( @@ -645,7 +643,7 @@ def add_workflow_args(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.8, - help="Minimal coverage of the alignment for two proteins to be in the same cluster", + help="Minimum coverage of the alignment for two proteins to be in the same cluster.", ) optional.add_argument( @@ -653,15 +651,14 @@ def add_workflow_args(parser: argparse.ArgumentParser): required=False, type=restricted_float, default=0.8, - help="Minimal identity percent for two proteins to be in the same cluster", + help="Minimum identity percentage for two proteins to be in the same cluster.", ) optional.add_argument( "--infer_singletons", required=False, action="store_true", - help="Use this option together with --clusters. " - "If a gene is not present in the provided clustering result file, " + help="Use this option together with --clusters. If a gene is not present in the provided clustering result file, " "it will be assigned to its own unique cluster as a singleton.", ) @@ -670,7 +667,7 @@ def add_workflow_args(parser: argparse.ArgumentParser): required=False, action="store_true", help="In the context of provided annotation, use this option to read pseudogenes. " - "(Default behavior is to ignore them)", + "(Default behavior is to ignore them).", ) optional.add_argument( @@ -688,7 +685,7 @@ def add_workflow_args(parser: argparse.ArgumentParser): "--no_defrag", required=False, action="store_true", - help="DO NOT Realign gene families to link fragments with their non-fragmented gene family.", + help="Do not realign gene families to link fragments with their non-fragmented gene family.", ) optional.add_argument( @@ -702,5 +699,5 @@ def add_workflow_args(parser: argparse.ArgumentParser): required=False, type=str, default=Path(tempfile.gettempdir()), - help="directory for storing temporary files", + help="Directory for storing temporary files.", ) diff --git a/ppanggolin/workflow/panModule.py b/ppanggolin/workflow/panModule.py index bac562a0..811c4da7 100644 --- a/ppanggolin/workflow/panModule.py +++ b/ppanggolin/workflow/panModule.py @@ -28,6 +28,10 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "panmodule", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = ( + "Easy workflow to run a pangenome analysis with module prediction" + ) + parser.category = "Basic" add_workflow_args(parser) diff --git a/ppanggolin/workflow/panRGP.py b/ppanggolin/workflow/panRGP.py index e2fd112a..defa9b4a 100644 --- a/ppanggolin/workflow/panRGP.py +++ b/ppanggolin/workflow/panRGP.py @@ -29,6 +29,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "panrgp", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Easy workflow to run a pangenome analysis with genomic islands and spots of insertion detection" + parser.category = "Basic" add_workflow_args(parser) diff --git a/ppanggolin/workflow/workflow.py b/ppanggolin/workflow/workflow.py index 269eab2f..01b71742 100644 --- a/ppanggolin/workflow/workflow.py +++ b/ppanggolin/workflow/workflow.py @@ -28,6 +28,8 @@ def subparser(sub_parser: argparse._SubParsersAction) -> argparse.ArgumentParser parser = sub_parser.add_parser( "workflow", formatter_class=argparse.RawTextHelpFormatter ) + parser.description = "Easy workflow to run a pangenome analysis in one go" + parser.category = "Basic" add_workflow_args(parser) diff --git a/ppanggolin_env.yaml b/ppanggolin_env.yaml index 388d7d2d..75060a9d 100644 --- a/ppanggolin_env.yaml +++ b/ppanggolin_env.yaml @@ -7,16 +7,17 @@ dependencies: - tqdm>=4.0.0,<5.0.0 - pytables>=3.8.0,<4.0.0 - pyrodigal>=3.0.0,<4.0.0 + - pyaragorn>=0.3.0 - networkx>=3.0.0,<4.0.0 - scipy>=1.0.0,<2.0.0 - plotly>=5.0.0,<6.0.0 - gmpy2>=2.0.0,<3.0.0 - pandas>=2.0.0,<3.0.0 - - numpy>1.24.0,<2.0.0 - - bokeh>=3.0.0,<4.0.0 + - numpy>1.24.0,<3.0.0 + - bokeh>=3.0.0,<3.4.0 - graph-tool>=2 + - gb-io>=0.4.0 # Tools that are not in Python - infernal=1 - - aragorn=1 - mmseqs2=15 - mafft=7 diff --git a/pyproject.toml b/pyproject.toml index 2791bc5c..f55ab57f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,19 +33,22 @@ classifiers=[ "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", "Topic :: Scientific/Engineering :: Bio-Informatics"] -requires-python = ">=3.9" +requires-python = ">=3.10,<3.14" dependencies = [ - "numpy>1.24.0,<2.0.0", + "numpy>1.24.0,<3.0.0", "pandas>=2.0.0,<3.0.0", "tqdm>=4.0.0,<5.0.0", "tables>=3.8.0,<4.0.0", "pyrodigal>=3.0.0,<4.0.0", + "pyaragorn>=0.3.0,<0.4.0", "networkx>=3.0.0,<4.0.0", "scipy>=1.0.0,<2.0.0", "plotly>=5.0.0,<6.0.0", "gmpy2>=2.0.0,<3.0.0", - "bokeh>=3.0.0,<4.0.0" + "bokeh>=3.0.0,<3.4.0", + "pyroaring>=1.0.0,<2.0.0", + "gb-io>=0.4.0", ] license = {file="LICENSE.txt"} @@ -61,9 +64,9 @@ doc = [ "sphinxcontrib.mermaid==0.9.2", ] test = [ - "pytest==7", + "pytest>=8.3.0,<9", "black==24.*", - "pytest-cov==6" + "pytest-cov>=6.0.0,<7" ] # diff --git a/testingDataset/expected_info_files/myannopang_info.yaml b/testingDataset/expected_info_files/myannopang_info.yaml index f283f7ab..cb4ce316 100644 --- a/testingDataset/expected_info_files/myannopang_info.yaml +++ b/testingDataset/expected_info_files/myannopang_info.yaml @@ -9,7 +9,7 @@ Status: Spots_Predicted: false Modules_Predicted: false translation_table: 11 - PPanGGOLiN_Version: 2.3.1 + PPanGGOLiN_Version: 2.3.2 Content: Genes: 47986 diff --git a/testingDataset/expected_info_files/mybasicpangenome_info.yaml b/testingDataset/expected_info_files/mybasicpangenome_info.yaml index 3e487ac7..17b31503 100644 --- a/testingDataset/expected_info_files/mybasicpangenome_info.yaml +++ b/testingDataset/expected_info_files/mybasicpangenome_info.yaml @@ -9,7 +9,7 @@ Status: Spots_Predicted: true Modules_Predicted: true translation_table: 11 - PPanGGOLiN_Version: 2.3.1 + PPanGGOLiN_Version: 2.3.2 Content: Genes: 45423 diff --git a/testingDataset/expected_info_files/panrgp_from_existing_cluster_info.yaml b/testingDataset/expected_info_files/panrgp_from_existing_cluster_info.yaml index 9e395458..88e14270 100644 --- a/testingDataset/expected_info_files/panrgp_from_existing_cluster_info.yaml +++ b/testingDataset/expected_info_files/panrgp_from_existing_cluster_info.yaml @@ -9,7 +9,7 @@ Status: Spots_Predicted: true Modules_Predicted: false translation_table: 11 - PPanGGOLiN_Version: 2.3.1 + PPanGGOLiN_Version: 2.3.2 Content: Genes: 47986 diff --git a/testingDataset/expected_info_files/stepbystep_info.yaml b/testingDataset/expected_info_files/stepbystep_info.yaml index b54ddae1..01f3ee41 100644 --- a/testingDataset/expected_info_files/stepbystep_info.yaml +++ b/testingDataset/expected_info_files/stepbystep_info.yaml @@ -9,7 +9,7 @@ Status: Spots_Predicted: true Modules_Predicted: true translation_table: 11 - PPanGGOLiN_Version: 2.3.1 + PPanGGOLiN_Version: 2.3.2 Content: Genes: 45423 diff --git a/tests/functional_tests/test_read_only_pangenome.py b/tests/functional_tests/test_read_only_pangenome.py new file mode 100644 index 00000000..064c3cef --- /dev/null +++ b/tests/functional_tests/test_read_only_pangenome.py @@ -0,0 +1,31 @@ +import os +import shutil + +import pytest + +from tests.utils.run_ppanggolin import run_ppanggolin_command +from tests.functional_tests.test_stepbystep import stepbystep_pangenome + + +def test_read_only_pangenome(stepbystep_pangenome, tmp_path): + """Commands that only read a pangenome must not require write access to it. + + Reference collections are distributed on read-only media, and making a + local copy read-only is the simplest way to keep a pangenome byte-identical + to the one it was downloaded from. + """ + pangenome = tmp_path / "pangenome.h5" + shutil.copy(stepbystep_pangenome, pangenome) + pangenome.chmod(0o444) + + if os.access(pangenome, os.W_OK): + # Running as root, or on a filesystem that ignores the mode. PyTables + # gates write modes on this very call, so the commands below would + # succeed even unfixed and the test would prove nothing. + pytest.skip("write access to the pangenome could not be revoked") + + run_ppanggolin_command(f"ppanggolin info -p {pangenome}") + + # Genome fluidity is already stored in the file, so this reads it back + # and prints it without recomputing anything. + run_ppanggolin_command(f"ppanggolin metrics -p {pangenome} --genome_fluidity") diff --git a/tests/functional_tests/test_rgp_clustering.py b/tests/functional_tests/test_rgp_clustering.py index 9c3e0b81..cdc6bfa5 100644 --- a/tests/functional_tests/test_rgp_clustering.py +++ b/tests/functional_tests/test_rgp_clustering.py @@ -21,7 +21,7 @@ ( "rgp_cluster_default", "ppanggolin rgp_cluster --pangenome {pangenome} --output {outdir}", - ["rgp_cluster.gexf", "rgp_cluster.tsv"], + ["rgp_cluster.graphml", "rgp_cluster.tsv"], ), ( "rgp_cluster_graph_formats", @@ -67,7 +67,7 @@ def test_rgp_cluster_with_metadata(pangenome_with_metadata, tmp_path): cmd = ( f"ppanggolin rgp_cluster --pangenome {pangenome_with_metadata} " - f"--output {outdir} --add_metadata" + f"--output {outdir} --add_metadata --graph_formats graphml gexf" ) run_ppanggolin_command(cmd) diff --git a/tests/functional_tests/test_write_pangenome.py b/tests/functional_tests/test_write_pangenome.py index 2850a4d4..cb27ad1c 100644 --- a/tests/functional_tests/test_write_pangenome.py +++ b/tests/functional_tests/test_write_pangenome.py @@ -1,5 +1,6 @@ # tests/functional_tests/test_stepbystep.py from pathlib import Path +import networkx as nx import pytest from tests.utils.run_ppanggolin import run_ppanggolin_command @@ -64,3 +65,17 @@ def test_stepbystep_outputs( fpath = Path(outdir) / fname assert fpath.exists(), f"Expected file {fname} not found after `{cmd}`" assert fpath.stat().st_size > 0, f"File {fname} is empty after `{cmd}`" + + if ".gexf" in fname: + # Existence is not enough: a malformed header still writes a + # non-empty file that no namespace-aware parser can read. + try: + graph = nx.read_gexf(fpath) + except Exception as err: + pytest.fail(f"{fname} was written but could not be parsed back: {err}") + assert graph.number_of_nodes() > 0, f"{fname} parsed but has no nodes" + # Fixing only the default namespace and leaving xmlns:viz behind + # still parses, but drops every colour and size. + assert any( + "viz" in data for _, data in graph.nodes(data=True) + ), f"{fname} parsed but its viz attributes were dropped" diff --git a/tests/unit_tests/annotate/test_annotate.py b/tests/unit_tests/annotate/test_annotate.py index 57b181c6..d57a3661 100644 --- a/tests/unit_tests/annotate/test_annotate.py +++ b/tests/unit_tests/annotate/test_annotate.py @@ -1,14 +1,11 @@ import pytest from pathlib import Path +import gb_io from ppanggolin.genome import Contig from ppanggolin.annotate.annotate import ( extract_positions, read_anno_file, - parse_contig_header_lines, - parse_gbff_by_contig, - parse_feature_lines, - parse_dna_seq_lines, read_org_gbff, combine_contigs_metadata, fix_partial_gene_coordinates, @@ -16,61 +13,127 @@ shift_end_coordinates, ) -from ppanggolin.annotate.synta import check_sequence_tuple, parse_fasta +from ppanggolin.annotate.synta import parse_fasta @pytest.mark.parametrize( "input_string, expected_positions, expected_complement, expected_partialgene_start, expected_partialgene_end", [ ( - "join(190..7695,7695..12071)", + gb_io.Join( + [ + gb_io.Range(189, 7695), + gb_io.Range(7694, 12071), + ] + ), [(190, 7695), (7695, 12071)], False, False, False, ), ( - "order(190..7695,7995..12071)", + gb_io.Order( + [ + gb_io.Range(189, 7695), + gb_io.Range(7994, 12071), + ] + ), [(190, 7695), (7995, 12071)], False, False, False, ), ( - "complement(join(4359800..4360707,4360707..4360962,1..100))", + gb_io.Complement( + gb_io.Join( + [ + gb_io.Range(4359799, 4360707), + gb_io.Range(4360706, 4360962), + gb_io.Range(0, 100), + ] + ) + ), [(4359800, 4360707), (4360707, 4360962), (1, 100)], True, False, False, ), ( - "complement(order(4359800..4360707,4360707..4360962,1..100))", + gb_io.Complement( + gb_io.Join( + [ + gb_io.Range(4359799, 4360707), + gb_io.Range(4360706, 4360962), + gb_io.Range(0, 100), + ] + ) + ), [(4359800, 4360707), (4360707, 4360962), (1, 100)], True, False, False, ), ( - "join(6835405..6835731,1..1218)", + gb_io.Join( + [ + gb_io.Range(6835404, 6835731), + gb_io.Range(0, 1218), + ] + ), [(6835405, 6835731), (1, 1218)], False, False, False, ), ( - "join(1375484..1375555,1375557..1376579)", + gb_io.Join( + [ + gb_io.Range(1375483, 1375555), + gb_io.Range(1375556, 1376579), + ] + ), [(1375484, 1375555), (1375557, 1376579)], False, False, False, ), - ("complement(6815492..6816265)", [(6815492, 6816265)], True, False, False), - ("6811501..6812109", [(6811501, 6812109)], False, False, False), - ("complement(6792573..>6795461)", [(6792573, 6795461)], True, False, True), - ("complement(<6792573..6795461)", [(6792573, 6795461)], True, True, False), - ("complement(<6792573..>6795461)", [(6792573, 6795461)], True, True, True), - ("join(1038313,1..1016)", [(1038313, 1038313), (1, 1016)], False, False, False), - ("1038313", [(1038313, 1038313)], False, False, False), + ( + gb_io.Complement(gb_io.Range(6815491, 6816265)), + [(6815492, 6816265)], + True, + False, + False, + ), + (gb_io.Range(6811500, 6812109), [(6811501, 6812109)], False, False, False), + ( + gb_io.Complement(gb_io.Range(6792572, 6795461, after=True)), + [(6792573, 6795461)], + True, + False, + True, + ), + ( + gb_io.Complement(gb_io.Range(6792572, 6795461, before=True)), + [(6792573, 6795461)], + True, + True, + False, + ), + ( + gb_io.Complement(gb_io.Range(6792572, 6795461, before=True, after=True)), + [(6792573, 6795461)], + True, + True, + True, + ), + ( + gb_io.Join([gb_io.Range(1038312, 1038313), gb_io.Range(0, 1016)]), + [(1038313, 1038313), (1, 1016)], + False, + False, + False, + ), + (gb_io.Range(1038312, 1038313), [(1038313, 1038313)], False, False, False), ], ) def test_extract_positions( @@ -91,38 +154,11 @@ def test_extract_positions( def test_extract_positions_with_wrong_positions_format(): with pytest.raises(ValueError): - extract_positions("join(1038313,1..1016") # string misses a closing parenthesis - - -def test_extract_positions_with_strange_chevrons(): - with pytest.raises(ValueError): - extract_positions( - "complement(join(4359800..>4360707,1..100))" - ) # chevron in inner position - with pytest.raises(ValueError): - extract_positions( - "complement(join(4359800..4360707,<1..100))" - ) # chevron in inner position - - with pytest.raises(ValueError): - extract_positions( - "complement(join(4359800..4360707,1..<100))" - ) # start chevron in ending position - - -def test_extract_positions_with_wrong_positions_format2(): - with pytest.raises(ValueError): - extract_positions("start..stop") # start and stop are not integer + extract_positions("join(1038313..1016") # string instead of object with pytest.raises(ValueError): - extract_positions( - "complement(join(start..6816265, 1..stop))" - ) # start and stop are not integer + extract_positions(None) # None instead of object with pytest.raises(ValueError): - extract_positions("start..stop") # start and stop are not integer - with pytest.raises(ValueError): - extract_positions( - "complement(join(start..6816265, 1..stop))" - ) # start and stop are not integer + extract_positions([1, 4]) # list instead of object @pytest.fixture @@ -214,170 +250,6 @@ def test_read_org_gbff(genome_data_with_joined_genes): assert genome.number_of_genes() == 917 -def test_gbff_header_parser(): - header_lines = [ - "LOCUS NC_022109 1041595 bp DNA circular CON 24-MAR-2017", - "DEFINITION Chlamydia trachomatis strain D/14-96 genome.", - "VERSION NC_022109.1", - "SOURCE Chlamydia trachomatis", - " ORGANISM Chlamydia trachomatis", - " Bacteria; Chlamydiae; Chlamydiales; Chlamydiaceae;", - " Chlamydia/Chlamydophila group; Chlamydia.", - ] - - parsed_header = parse_contig_header_lines(header_lines) - - assert parsed_header == { - "LOCUS": "NC_022109 1041595 bp DNA circular CON 24-MAR-2017", - "DEFINITION": "Chlamydia trachomatis strain D/14-96 genome.", - "VERSION": "NC_022109.1", - "SOURCE": "Chlamydia trachomatis", - "ORGANISM": "Chlamydia trachomatis\nBacteria; Chlamydiae; Chlamydiales; Chlamydiaceae;\nChlamydia/Chlamydophila group; Chlamydia.", - } - - -# Define test data -@pytest.fixture -def sample_gbff_path(tmp_path): - gbff_content = """LOCUS NC_022109 1041595 bp DNA circular CON 24-MAR-2017 -DEFINITION Chlamydia trachomatis strain D/14-96 genome. -ACCESSION NC_022109 -VERSION NC_022109.1 -KEYWORDS RefSeq. -SOURCE Chlamydia trachomatis -FEATURES Location/Qualifiers - source 1..1041595 - /organism="Chlamydia trachomatis" - /mol_type="genomic DNA" -ORIGIN - 1 aaaccgggtt - 11 ccaaatttgg -// -LOCUS NC_022110 2041595 bp DNA circular CON 24-MAR-2017 -DEFINITION Another genome. -KEYWORDS RefSeq. -SOURCE Chlamydia trachomatis - ORGANISM Chlamydia trachomatis - Bacteria; Chlamydiae; Chlamydiales; Chlamydiaceae; - Chlamydia/Chlamydophila group; Chlamydia. -FEATURES Location/Qualifiers - source 1..2041595 - /organism="Chlamydia trachomatis" - /mol_type="genomic DNA" -ORIGIN - 1 aaaccgggtt - 11 ccaaatttgg - 21 ggcccctttt -// -""" - - gbff_file_path = tmp_path / "sample.gbff" - with open(gbff_file_path, "w") as f: - f.write(gbff_content) - return gbff_file_path - - -# Define test cases -def test_parse_gbff_by_contig(sample_gbff_path): - contigs = list(parse_gbff_by_contig(sample_gbff_path)) - - assert len(contigs) == 2 - - # Check first contig - header_1, feature_1, sequence_1 = contigs[0] - assert len(header_1) == 6 - assert len(list(feature_1)) == 1 - assert sequence_1 == "AAACCGGGTTCCAAATTTGG" - - # Check second contig - header_2, feature_2, sequence_2 = contigs[1] - assert len(header_2) == 5 - assert list(feature_2) == [ - { - "feature_type": "source", - "location": "1..2041595", - "organism": "Chlamydia trachomatis", - "mol_type": "genomic DNA", - } - ] - - assert sequence_2 == "AAACCGGGTTCCAAATTTGGGGCCCCTTTT" - - -# Define test cases -@pytest.mark.parametrize( - "input_lines, expected_output", - [ - ( - [ - " gene 123..456", - " /locus_tag=ABC123", - " /note=Some note", - " CDS 789..1011", - " /protein_id=DEF456", - " /translation=ATGCTAGCATCG", - ], - [ - { - "feature_type": "gene", - "location": "123..456", - "locus_tag": "ABC123", - "note": "Some note", - }, - { - "feature_type": "CDS", - "location": "789..1011", - "protein_id": "DEF456", - "translation": "ATGCTAGCATCG", - }, - ], - ), - ( - [ - " gene 123..456", - " /locus_tag=ABC123", - " /note=Some note", - " CDS 789..1011", - ' /protein_id="DEF456"', - ' /translation="ATGCTAGCATCG"', - " gene 789..1011", - " /locus_tag=DEF789", - " /note=Another note", - ], - [ - { - "feature_type": "gene", - "location": "123..456", - "locus_tag": "ABC123", - "note": "Some note", - }, - { - "feature_type": "CDS", - "location": "789..1011", - "protein_id": "DEF456", - "translation": "ATGCTAGCATCG", - }, - { - "feature_type": "gene", - "location": "789..1011", - "locus_tag": "DEF789", - "note": "Another note", - }, - ], - ), - # Add more test cases as needed - ], -) -def test_parse_feature_lines(input_lines, expected_output): - assert list(parse_feature_lines(input_lines)) == expected_output - - -def test_parse_dna_seq_lines(): - lines = [" 1 aaacc gggtt", " 11 ccaaa tttgg", " 21 ggccc ctttt"] - - assert parse_dna_seq_lines(lines) == "AAACCGGGTTCCAAATTTGGGGCCCCTTTT" - - def test_combine_contigs_metadata(): contig1, contig2, contig3 = ( Contig(1, "contig1"), @@ -535,22 +407,6 @@ def test_shift_end_coordinates(coordinates, shift, expected): assert result == expected -def test_check_sequence_tuple_valid(): - name, sequence = check_sequence_tuple("seq1", "ATGC") - assert name == "seq1" - assert sequence == "ATGC" - - -def test_check_sequence_tuple_empty_name(): - with pytest.raises(ValueError): - check_sequence_tuple("", "ATGC") - - -def test_check_sequence_tuple_empty_sequence(): - with pytest.raises(ValueError): - check_sequence_tuple("seq1", "") - - def test_parse_fasta_valid(): fasta_data = ">seq1\nATGC\n>seq2\nGCTA" @@ -561,11 +417,32 @@ def test_parse_fasta_valid(): def test_parse_fasta_empty_sequence(): fasta_data = ">seq1\n>seq2\nGCTA" - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match="Malformed FASTA record for contig 'seq1'|empty sequence" + ): list(parse_fasta(fasta_data.split("\n"))) def test_parse_fasta_no_header(): fasta_data = "seq1\nATGC\nseq2\nGCTA".split("\n") - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match="Malformed FASTA content|header line starting with '>'" + ): + list(parse_fasta(fasta_data)) + + +def test_parse_fasta_empty_header_name(): + fasta_data = ">\nATGC\n".split("\n") + with pytest.raises( + ValueError, match="Malformed FASTA header|contains no contig name" + ): list(parse_fasta(fasta_data)) + + +def test_parse_fasta_includes_source_path_in_error(tmp_path: Path): + fasta_path = tmp_path / "bad_contigs.fa" + fasta_path.write_text(">seq1\n>seq2\nGCTA\n", encoding="utf-8") + + with open(fasta_path, "r", encoding="utf-8") as handle: + with pytest.raises(ValueError, match=str(fasta_path)): + list(parse_fasta(handle)) diff --git a/tests/unit_tests/region/test_rgp_cluster.py b/tests/unit_tests/region/test_rgp_cluster.py index d8b914a7..f34b5d72 100644 --- a/tests/unit_tests/region/test_rgp_cluster.py +++ b/tests/unit_tests/region/test_rgp_cluster.py @@ -1,236 +1,137 @@ #! /usr/bin/env python3 import pytest -from random import randint -from typing import Generator, Set -from ppanggolin.RGP import rgp_cluster -from ppanggolin.genome import Gene, Contig, Organism -from ppanggolin.geneFamily import GeneFamily -from ppanggolin.region import Region -from ppanggolin.RGP.rgp_cluster import IdenticalRegions +from pyroaring import BitMap + +from ppanggolin.RGP.rgp_cluster import RGPClustering, RegionProxy, RGPInfo, Contig @pytest.fixture -def genes() -> Generator[Set[Gene], None, None]: - """Create a set of genes to fill gene families""" - organism = Organism("organism") - contig = Contig(0, "contig") - genes = [] - for i in range(randint(11, 20)): - gene = Gene(f"gene_{str(i)}") - gene.fill_annotations( - start=10 * i + 1, stop=10 * (i + 1), strand="+", position=i, genetic_code=4 - ) - gene.fill_parents(organism, contig) - contig[gene.start] = gene - genes.append(gene) - return genes +def rgp_a(): + return RegionProxy( + ID=1, + name="RGP_A", + families=BitMap([1, 2, 3, 4, 5]), + is_contig_border=True, + is_whole_contig=False, + ) @pytest.fixture -def families(genes) -> Generator[Set[GeneFamily], None, None]: - """Create a set of gene families fill with genes to test edges""" - families = set() - genes = list(genes) - nb_families = randint(9, 20) - nb_genes_per_family = len(genes) // nb_families - idx_fam = 1 - while idx_fam < nb_families: - family = GeneFamily(idx_fam, f"family_{idx_fam}") - idx_genes = 0 - while idx_genes < nb_genes_per_family: - gene = genes[(idx_fam - 1) * nb_genes_per_family + idx_genes] - family.add(gene) - gene.family = family - idx_genes += 1 - families.add(family) - idx_fam += 1 - # last family fill with all the gene left - family = GeneFamily(idx_fam, f"family_{idx_fam}") - idx_genes = (idx_fam - 1) * nb_genes_per_family - while idx_genes < len(genes): - gene = genes[idx_genes] - family.add(gene) - gene.family = family - idx_genes += 1 - families.add(family) - yield families +def rgp_b(): + return RegionProxy( + ID=2, + name="RGP_B", + families=BitMap([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), + is_contig_border=False, + is_whole_contig=False, + ) @pytest.fixture -def identical_rgps(genes, families) -> Generator[Set[Region], None, None]: - """Create a set of identical rgps""" - identical_rgps = set() - for i in range(1, randint(6, 21)): - rgp = Region(f"RGP_{i}") - # the three rgp have the gene content. - # in terms of family they are identical - for gene in genes: - rgp[gene.position] = gene - identical_rgps.add(rgp) - yield identical_rgps - - -class TestIdenticalRegions: - def test_init_with_valid_inputs(self, identical_rgps, families): - """Tests that the IdenticalRegions object is initialized correctly with valid inputs.""" - is_contig_border = True - identical_regions = IdenticalRegions( - "IdenticalRegions", identical_rgps, families, is_contig_border +def clustering(tmp_path): + clustering = RGPClustering(tmp_path / "pangenome.h5") + clustering._fam_to_modules = {} + clustering._rgp_to_nb_genes = {} + clustering._rgp_contig_to_info = { + "contig_1": Contig( + organism="organism_1", + is_circular=False, + idx=0, + name="contig_1", ) + } + return clustering + + +@pytest.mark.parametrize( + ("families_a", "families_b", "operation", "expected"), + [ + ( + BitMap([1, 2, 3, 4, 5]), + BitMap([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), + min, + 1.0, + ), + ( + BitMap([1, 2, 3, 4, 5]), + BitMap([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), + max, + 0.5, + ), + ], +) +def test_grr(clustering, families_a, families_b, operation, expected): + assert clustering._grr(families_a, families_b, operation) == expected + + +def test_rgp_metric(clustering, rgp_a, rgp_b): + metric = clustering._rgp_metric( + rgp_a, + rgp_b, + grr_cutoff=0, + metric="min_grr", + ) - assert identical_regions.name == "IdenticalRegions" - assert identical_regions.rgps == identical_rgps - assert identical_regions.families == families - assert identical_regions.is_contig_border == is_contig_border - - @pytest.mark.parametrize("wrong_type", ["string", 1, 0.8, list(), dict()]) - def test_init_with_identical_rgps_not_isintance_set(self, wrong_type, families): - """Tests that the IdenticalRegions object cannot be initialized with a not instance set for identical_rgps.""" - with pytest.raises(TypeError): - IdenticalRegions("IdenticalRegions", wrong_type, families, True) - - def test_init_with_rgp_is_not_instance_region_in_identical_rgps( - self, identical_rgps, families - ): - """Tests that the IdenticalRegions object raise TypeError if one element is not instance Region.""" - with pytest.raises(TypeError): - IdenticalRegions( - "IdenticalRegions", identical_rgps.union({1}), families, True - ) - - def test_init_with_empty_identical_rgps(self, families): - """Tests that the IdenticalRegions object cannot be initialized with an empty set of identical regions.""" - with pytest.raises(ValueError): - IdenticalRegions("IdenticalRegions", set(), families, True) - - @pytest.mark.parametrize("wrong_type", ["string", 1, 0.8, list(), dict()]) - def test_init_with_families_not_isintance_set(self, wrong_type, identical_rgps): - """Tests that the IdenticalRegions object cannot be initialized with a not instance set.""" - with pytest.raises(TypeError): - IdenticalRegions("IdenticalRegions", identical_rgps, wrong_type, True) - - def test_init_with_family_is_not_instance_genefamilies_in_families( - self, identical_rgps, families - ): - """Tests that the IdenticalRegions object raise TypeError if one element is not instance Region.""" - with pytest.raises(TypeError): - IdenticalRegions( - "IdenticalRegions", identical_rgps, families.union({1}), True - ) - - def test_init_with_empty_families(self, identical_rgps): - """Tests that the IdenticalRegions object cannot be initialized with an empty set of identical regions.""" - with pytest.raises(ValueError): - IdenticalRegions("IdenticalRegions", identical_rgps, set(), True) - - def test_eq_with_equal_identical_regions(self): - """Tests that the __eq__ method returns Trueè when comparing two IdenticalRegions objects that have the same families, - identical regions, and contig border status. - """ - rgp1 = Region("RGP1") - rgp2 = Region("RGP2") - family1 = GeneFamily(1, "Family1") - family2 = GeneFamily(2, "Family2") - identical_rgps1 = {rgp1, rgp2} - identical_rgps2 = {rgp1, rgp2} - families1 = {family1, family2} - families2 = {family1, family2} - is_contig_border = True - - identical_regions1 = IdenticalRegions( - "IdenticalRegions", identical_rgps1, families1, is_contig_border - ) - identical_regions2 = IdenticalRegions( - "IdenticalRegions", identical_rgps2, families2, is_contig_border - ) + assert metric is not None + assert metric.min_grr == 1.0 + assert metric.max_grr == 0.5 + assert metric.incomplete_aware_grr == 1.0 + assert metric.shared_family == 5 - assert identical_regions1 == identical_regions2 - - def test_eq_with_non_identical_regions(self): - """Tests that the __eq__ method returns False when comparing - two IdenticalRegions objects that have different families. - """ - rgp1 = Region("RGP1") - rgp2 = Region("RGP2") - family1 = GeneFamily(1, "Family1") - family2 = GeneFamily(2, "Family2") - identical_rgps1 = {rgp1, rgp2} - identical_rgps2 = {rgp1, rgp2} - families1 = {family1, family2} - families2 = {family1} - is_contig_border = True - - identical_regions1 = IdenticalRegions( - "IdenticalRegions", identical_rgps1, families1, is_contig_border - ) - identical_regions2 = IdenticalRegions( - "IdenticalRegions", identical_rgps2, families2, is_contig_border - ) - assert identical_regions1 != identical_regions2 +def test_rgp_metric_below_cutoff(clustering, rgp_a, rgp_b): + metric = clustering._rgp_metric( + rgp_a, + rgp_b, + grr_cutoff=1.0, + metric="max_grr", + ) + + assert metric is None -def test_compute_grr(): - """Tests that compute_grr returns the correct value when there is a non-zero intersection between families""" - set1 = {1, 2, 3, 4, 5} - set2 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10} +def test_rgp_metric_for_complete_rgps(clustering, rgp_a, rgp_b): + rgp_a.is_contig_border = False + rgp_a.is_whole_contig = False - assert rgp_cluster.compute_grr(set1, set2, min) == 1.0 - assert rgp_cluster.compute_grr(set1, set2, max) == 0.5 + rgp_b.is_contig_border = False + rgp_b.is_whole_contig = False + metric = clustering._rgp_metric( + rgp_a, + rgp_b, + grr_cutoff=0, + metric="max_grr", + ) + + assert metric is not None + assert metric.min_grr == 1.0 + assert metric.max_grr == 0.5 + assert metric.incomplete_aware_grr == 0.5 + assert metric.shared_family == 5 -def test_dereplicate_rgp(identical_rgps): - list_identical_rgps = list(identical_rgps) - rgp1 = list_identical_rgps[0] - assert rgp_cluster.dereplicate_rgp({rgp1}) == [rgp1] - identical_region_obj = rgp_cluster.IdenticalRegions( - name="identical_rgps_0", - identical_rgps=identical_rgps, - families=set(list_identical_rgps[0].families), +def test_construct_single(clustering): + rgp = RGPInfo( + name="RGP_1", + families_ids=BitMap([1, 2, 3]), + families=set(), + contig="contig_1", is_contig_border=True, + is_whole_contig=False, ) - assert rgp_cluster.dereplicate_rgp(rgps=identical_rgps)[0] == identical_region_obj - - -def test_compute_rgp_metric(genes, families): - RGP_a = Region("A") - RGP_b = Region("B") - list_genes = sorted(genes, key=lambda x: x.position) - - for g in list_genes[:8]: - RGP_a[g.position] = g - for g in list_genes[3:7]: - RGP_b[g.position] = g - - assert RGP_a.is_contig_border - assert not RGP_b.is_contig_border - - shared_families = len(set(RGP_a.families).intersection(set(RGP_b.families))) - expected_grr = ( - RGP_a.ID, - RGP_b.ID, - { - "incomplete_aware_grr": shared_families - / min(len(set(RGP_a.families)), len(set(RGP_b.families))), - "min_grr": shared_families - / min(len(set(RGP_a.families)), len(set(RGP_b.families))), - "max_grr": shared_families - / max(len(set(RGP_a.families)), len(set(RGP_b.families))), - "shared_family": shared_families, - }, - ) - # min_grr - min_result = rgp_cluster.compute_rgp_metric(RGP_a, RGP_b, 0, "min_grr") - assert min_result == expected_grr - # incomplete_aware_grr: same as min grr as rgp1 is incomplete - incomplete_aware_result = rgp_cluster.compute_rgp_metric( - RGP_a, RGP_b, 0, "incomplete_aware_grr" - ) + clustering._rgp_to_nb_genes["RGP_1"] = 10 - assert incomplete_aware_result == expected_grr + region = clustering._construct_single(1, rgp) - # max grr is below cutoff so None is returned - assert rgp_cluster.compute_rgp_metric(RGP_a, RGP_b, 1000, "max_grr") is None + assert region.ID == 1 + assert region.name == "RGP_1" + assert region.families == BitMap([1, 2, 3]) + assert region.organism == "organism_1" + assert region.contig == "contig_1" + assert region.length == 10 + assert region.is_contig_border + assert not region.is_whole_contig + assert not region.is_identical_region diff --git a/tests/unit_tests/test_cli_reference.py b/tests/unit_tests/test_cli_reference.py new file mode 100644 index 00000000..6825ab51 --- /dev/null +++ b/tests/unit_tests/test_cli_reference.py @@ -0,0 +1,94 @@ +import argparse +import importlib.util +from pathlib import Path + +from ppanggolin import SUBCOMMAND_TO_SUBPARSER +from ppanggolin.main import build_parser + + +def test_build_parser_is_available_without_sys_argv(): + parser = build_parser() + + assert parser is not None + assert parser.prog == "ppanggolin" + assert "all" in parser._subparsers._group_actions[0].choices + assert "info" in parser._subparsers._group_actions[0].choices + assert "utils" in parser._subparsers._group_actions[0].choices + + +def test_build_parser_does_not_execute_analysis(monkeypatch): + import ppanggolin.workflow.all as workflow_all + + def boom(*args, **kwargs): + raise AssertionError( + "analysis launch should not be called while building the parser" + ) + + monkeypatch.setattr(workflow_all, "launch", boom) + + parser = build_parser() + assert parser is not None + + +def test_subparsers_own_command_metadata(): + parser = argparse.ArgumentParser(prog="ppanggolin") + subparsers = parser.add_subparsers(dest="subcommand") + + all_parser = SUBCOMMAND_TO_SUBPARSER["all"](subparsers) + info_parser = SUBCOMMAND_TO_SUBPARSER["info"](subparsers) + + assert all_parser.category == "Basic" + assert all_parser.description == "Easy workflow to run all possible analysis" + assert info_parser.category == "Output" + assert ( + info_parser.description + == "Prints information about a given pangenome graph file." + ) + + +def test_top_level_help_lists_subcommands(): + parser = build_parser() + help_text = parser.format_help() + + assert "subcommands:" in help_text + assert "all" in help_text + assert "info" in help_text + assert "utils" in help_text + assert "All of the following subcommands have their own set of options" in help_text + assert "Basic:" in help_text + + +def test_generated_reference_contains_cli_data(tmp_path): + script_path = ( + Path(__file__).resolve().parents[2] + / "docs" + / "_scripts" + / "generate_command_reference.py" + ) + spec = importlib.util.spec_from_file_location( + "generate_command_reference", script_path + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + output_path = tmp_path / "command_reference.md" + rendered = module.generate_reference(output_path) + + assert "# Command-line reference" in rendered + assert "## `ppanggolin all`" in rendered + assert "## `ppanggolin info`" in rendered + assert "## `ppanggolin utils`" in rendered + assert "## Basic" in rendered + assert "Easy workflow to run all possible analysis" in rendered + assert "## Utility command" in rendered + assert "Helper side commands" in rendered + assert "--cpu" in rendered + assert "--pangenome" in rendered + assert "False" in rendered + assert "Choices" in rendered + assert "bool" in rendered + assert "float" in rendered + assert "Input arguments for ppanggolin all" in rendered + assert "Optional arguments for ppanggolin all" in rendered + assert "Common arguments for ppanggolin utils" in rendered + assert "ppanggolin_context__" in rendered diff --git a/tests/unit_tests/utils/test_utilities.py b/tests/unit_tests/utils/test_utilities.py index 68ce840a..c65d333f 100644 --- a/tests/unit_tests/utils/test_utilities.py +++ b/tests/unit_tests/utils/test_utilities.py @@ -13,6 +13,7 @@ write_compressed_or_not, has_non_ascii, replace_non_ascii, + parse_input_paths_file, ) @@ -197,3 +198,83 @@ def test_has_non_ascii(input_string, expected): ) def test_replace_non_ascii(input_string, replacement, expected): assert replace_non_ascii(input_string, replacement) == expected + + +def test_parse_input_paths_file_valid_file(tmp_path: Path): + fasta = tmp_path / "genome1.fa" + fasta.write_text(">seq\nACGT\n") + + path_list = tmp_path / "genomes.tsv" + path_list.write_text( + "# comment\n" "genome1\tgenome1.fa\tcontig_1\n" "\n" "genome2\tgenome2.fa\n", + encoding="utf-8", + ) + (tmp_path / "genome2.fa").write_text(">seq\nACGT\n") + + parsed = parse_input_paths_file(path_list) + + assert list(parsed) == ["genome1", "genome2"] + assert parsed["genome1"]["path"] == fasta + assert parsed["genome1"]["circular_contigs"] == ["contig_1"] + assert parsed["genome2"]["circular_contigs"] == [] + + +def test_parse_input_paths_file_rejects_missing_field(tmp_path: Path): + path_list = tmp_path / "genomes.tsv" + path_list.write_text("genome1\n", encoding="utf-8") + + with pytest.raises(ValueError, match="at least 2 tab-separated columns"): + parse_input_paths_file(path_list) + + +def test_parse_input_paths_file_rejects_duplicate_names(tmp_path: Path): + (tmp_path / "genome1.fa").write_text(">seq\nACGT\n") + path_list = tmp_path / "genomes.tsv" + path_list.write_text( + "genome1\tgenome1.fa\n" "genome1\tgenome1.fa\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="Duplicate genome name"): + parse_input_paths_file(path_list) + + +def test_parse_input_paths_file_rejects_trimmed_duplicate_names(tmp_path: Path): + (tmp_path / "genome1.fa").write_text(">seq\nACGT\n") + path_list = tmp_path / "genomes.tsv" + path_list.write_text( + "genome1 \tgenome1.fa\n" "genome1\tgenome1.fa\n", + encoding="utf-8", + ) + + with pytest.raises( + ValueError, match="Duplicate genome name 'genome1'.*after trimming" + ): + parse_input_paths_file(path_list) + + +def test_parse_input_paths_file_rejects_space_in_name(tmp_path: Path): + (tmp_path / "genome1.fa").write_text(">seq\nACGT\n") + path_list = tmp_path / "genomes.tsv" + path_list.write_text("genome 1\tgenome1.fa\n", encoding="utf-8") + + with pytest.raises(ValueError, match="must not contain whitespace"): + parse_input_paths_file(path_list) + + +def test_parse_input_paths_file_rejects_directory_path(tmp_path: Path): + directory = tmp_path / "genome_dir" + directory.mkdir() + path_list = tmp_path / "genomes.tsv" + path_list.write_text("genome1\tgenome_dir\n", encoding="utf-8") + + with pytest.raises(ValueError, match="resolves to a directory, not a file"): + parse_input_paths_file(path_list) + + +def test_parse_input_paths_file_rejects_missing_file(tmp_path: Path): + path_list = tmp_path / "genomes.tsv" + path_list.write_text("genome1\tmissing.fa\n", encoding="utf-8") + + with pytest.raises(FileNotFoundError, match="does not exist"): + parse_input_paths_file(path_list)