diff --git a/docs/_ext/github_readme.py b/docs/_ext/github_readme.py new file mode 100644 index 00000000..92ac2623 --- /dev/null +++ b/docs/_ext/github_readme.py @@ -0,0 +1,169 @@ +"""Sphinx extension to reuse the GitHub README as the documentation landing page. + +README.md is written for GitHub. When it is pulled into the docs with MyST's ``{include}`` +directive, this extension replaces the directive by the README content and: + +- rewrites GitHub alerts (``> [!NOTE]``) into MyST admonitions; +- warns about MyST-only syntax, which GitHub would display as raw text. + +The other GitHub-flavored features (bare URL autolinks, strikethrough, task lists, mermaid +fences) are handled by MyST extensions enabled in ``conf.py``. + +This module must stay importable without Sphinx so the README can be checked by the test suite. +""" + +import os +import re +from pathlib import Path + +README_NAME = "README.md" + +GITHUB_ALERT = re.compile( + r"^> \[!(?PNOTE|TIP|IMPORTANT|WARNING|CAUTION)\][ \t]*\n(?P(?:^>.*(?:\n|$))*)", + flags=re.MULTILINE, +) + +FENCE = re.compile(r"^ {0,3}(?P`{3,}|~{3,}|:{3,})(?P.*)$") +# Scanned left to right, so a role written inside an inline code span is consumed as code +MYST_ROLE_OR_CODE = re.compile(r"(?P\{[\w:.+-]+\}`[^`]+`)|`+[^`]*`+") +MYST_INLINE_SYNTAX = { + "MyST target": re.compile(r"^\s*\([^()\s]+\)=\s*$"), + "MyST comment": re.compile(r"^\s*%"), + "MyST block break": re.compile(r"^\s*\+\+\+"), +} + +MYST_INCLUDE = re.compile( + r"^(?P`{3,}|:{3,})\{include\}(?P[^\n]+)\n(?P(?::[^\n]*\n)*)(?P=fence)[ \t]*$", + flags=re.MULTILINE, +) +MARKDOWN_IMAGE = re.compile( + r"(?P!\[[^\]]*\])\((?P[^)\s]+)(?P[^)]*)\)" +) +HTML_IMAGE = re.compile( + r"(?P<before><img\b[^>]*?\bsrc=)(?P<quote>[\"'])(?P<target>[^\"']+)(?P=quote)", + flags=re.IGNORECASE, +) + + +def github_alerts_to_myst(text: str) -> str: + """Convert GitHub alert blockquotes into MyST colon-fence admonitions. + + Args: + text: Markdown content using GitHub alert syntax. + + Returns: + The same content with every alert replaced by a MyST admonition. + """ + + def _replace(match: re.Match) -> str: + body = re.sub(r"^> ?", "", match["body"], flags=re.MULTILINE).rstrip("\n") + return f":::{{{match['kind'].lower()}}}\n{body}\n:::\n" + + return GITHUB_ALERT.sub(_replace, text) + + +def find_myst_only_syntax(text: str) -> list[tuple[int, str]]: + """Find MyST syntax that GitHub cannot render. + + Content of regular code blocks is ignored, so documenting MyST syntax inside a code block is fine. + + Args: + text: Markdown content meant to be rendered by GitHub. + + Returns: + A list of ``(line_number, description)`` tuples, one per offending line (1-indexed). + """ + issues = [] + open_fence = None + for line_number, line in enumerate(text.splitlines(), start=1): + fence = FENCE.match(line) + if open_fence is not None: + if ( + fence + and fence["marker"][0] == open_fence[0] + and len(fence["marker"]) >= len(open_fence) + ): + if not fence["info"].strip(): + open_fence = None + continue + if fence: + marker, info = fence["marker"], fence["info"].strip() + if marker.startswith(":"): + issues.append((line_number, f"MyST colon fence '{line.strip()}'")) + elif info.startswith("{"): + issues.append((line_number, f"MyST directive '{line.strip()}'")) + open_fence = marker + continue + if any(match["role"] for match in MYST_ROLE_OR_CODE.finditer(line)): + issues.append((line_number, f"MyST role in '{line.strip()}'")) + for name, pattern in MYST_INLINE_SYNTAX.items(): + if pattern.match(line): + issues.append((line_number, f"{name} '{line.strip()}'")) + return issues + + +def rebase_images(text: str, readme_dir: Path, doc_dir: Path) -> str: + """Make relative image paths of the README relative to the including document. + + Reproduces the ``:relative-images:`` option of MyST's ``{include}`` directive, which only + handles Markdown images, and also applies it to the ``src`` of HTML ``<img>`` tags. Raw HTML is + copied as is to the output, so an HTML image only shows up if its file is part of the output + too (for example, in ``html_static_path``). + + Args: + text: README content. + readme_dir: Directory containing the README. + doc_dir: Directory containing the document that includes the README. + + Returns: + The content with every relative image path rebased on ``doc_dir``. + """ + + def _rebase(target: str) -> str: + if re.match(r"^([a-z][a-z0-9+.-]*:|/|#)", target, flags=re.IGNORECASE): + return target + return Path(os.path.relpath(readme_dir / target, doc_dir)).as_posix() + + text = MARKDOWN_IMAGE.sub( + lambda m: f"{m['alt']}({_rebase(m['target'])}{m['title']})", text + ) + return HTML_IMAGE.sub( + lambda m: f"{m['before']}{m['quote']}{_rebase(m['target'])}{m['quote']}", text + ) + + +def expand_readme_include(app, docname: str, source: list[str]) -> None: + """Sphinx ``source-read`` handler inlining the README converted from GitHub to MyST syntax. + + MyST's ``{include}`` directive reads the file itself, so the README cannot be converted when it + is included. The directive is instead replaced by the converted README content before parsing. + """ + from sphinx.util import logging + + logger = logging.getLogger(__name__) + doc_path = Path(app.env.doc2path(docname)) + + def _replace(match: re.Match) -> str: + readme_path = (doc_path.parent / match["path"].strip()).resolve() + if readme_path.name != README_NAME: + return match[0] + app.env.note_dependency(str(readme_path)) + text = readme_path.read_text(encoding="utf-8") + for line_number, issue in find_myst_only_syntax(text): + logger.warning( + "%s will not render on GitHub, use GitHub-flavored Markdown instead", + issue, + location=f"{readme_path}:{line_number}", + ) + text = github_alerts_to_myst(text) + if re.search(r"^:relative-images:", match["options"], flags=re.MULTILINE): + text = rebase_images(text, readme_path.parent, doc_path.parent) + return text + + source[0] = MYST_INCLUDE.sub(_replace, source[0]) + + +def setup(app): + """Register the extension in Sphinx.""" + app.connect("source-read", expand_readme_include) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/conf.py b/docs/conf.py index 389943a4..1a6b6a1a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,6 +14,8 @@ import sys from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent / "_ext")) + def generate_command_reference(_app): script = ( @@ -38,7 +40,6 @@ def setup(app): open(Path(__file__).resolve().parents[1] / "VERSION").read().rstrip() ) # Get release number in the VERSION file - # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be @@ -52,8 +53,15 @@ def setup(app): "sphinx.ext.autodoc", "sphinx_search.extension", "sphinxcontrib.mermaid", + "github_readme", # Local extension (_ext/): renders the GitHub README as the landing page ] +# These extensions mirror GitHub-flavored Markdown so README.md renders the same in both places +myst_enable_extensions = ["linkify", "strikethrough", "tasklist"] +myst_fence_as_directive = ["mermaid"] +myst_linkify_fuzzy_links = ( + False # Like GitHub, only autolink http(s):// and www. URLs, not "README.md" +) source_suffix = {".md": "markdown"} diff --git a/docs/index.md b/docs/index.md index ab81ca19..f96f6472 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,48 +3,11 @@ % You can adapt this file completely to your liking, but it should at least % contain the root `toctree` directive. - -# PPanGGOLiN: Depicting microbial species diversity via a Partitioned PanGenome Graph Of Linked Neighbors - - -[![Actions](https://img.shields.io/github/actions/workflow/status/labgem/PPanGGOLiN/main.yml?branch=master&label=CI&logo=github)](https://github.com/labgem/PPanGGOLiN/actions/workflows/main.yml) -[![License](https://anaconda.org/bioconda/ppanggolin/badges/license.svg)](http://www.cecill.info/licences.fr.html) -[![Bioconda](https://img.shields.io/conda/vn/bioconda/ppanggolin?style=flat-square&maxAge=3600&logo=anaconda)](https://anaconda.org/bioconda/ppanggolin) -[![PyPI version](https://badge.fury.io/py/PPanGGOLiN.svg?cache-control=no-cache)](https://pypi.org/project/PPanGGOLiN/) -[![Source](https://img.shields.io/badge/source-GitHub-303030.svg?maxAge=2678400&style=flat-square)](https://github.com/labgem/ppanggolin/) -[![GitHub issues](https://img.shields.io/github/issues/labgem/ppanggolin.svg?style=flat-square&maxAge=600)](https://github.com/labgem/ppanggolin/issues) -[![Docs](https://img.shields.io/readthedocs/ppanggolin/latest?style=flat-square&maxAge=600)](https://ppanggolin.readthedocs.io) -[![Downloads](https://anaconda.org/bioconda/ppanggolin/badges/downloads.svg)](https://bioconda.github.io/recipes/ppanggolin/README.html#download-stats) - - -```{image} _static/logo.png -:alt: ppangolin logo -:align: center -:width: 437 +```{include} ../README.md +:relative-images: ``` -<br> - - -**PPanGGOLiN** -([Gautreau et al. 2020](https://doi.org/10.1371/journal.pcbi.1007732)) is a software suite used to create and manipulate prokaryotic pangenomes from a set of either assembled genomic DNA sequences or provided genome annotations. -It is designed to scale up to tens of thousands of genomes. -It has the specificity to partition the pangenome using a statistical approach rather than using fixed thresholds which gives it the ability to work with low-quality data such as *Metagenomic Assembled Genomes (MAGs)* or *Single-cell Amplified Genomes (SAGs)* thus taking advantage of large scale environmental studies and letting users study the pangenome of uncultivable species. - -**PPanGGOLiN** builds pangenomes through a graphical model and a statistical method to partition gene families in persistent, shell and cloud genomes. -It integrates both information on protein-coding genes and their genomic neighborhood to build a graph of gene families where each node is a gene family, and each edge is a relation of genetic contiguity. -The partitioning method promotes that two gene families that are consistent neighbors in the graph are more likely to belong to the same partition. -It results in a Partitioned Pangenome Graph (PPG) made of persistent, shell and cloud nodes drawing genomes on rails like a subway map to help biologists navigate the great diversity of microbial life. - - -Moreover, the panRGP method ([Bazin et al. 2020](https://doi.org/10.1093/bioinformatics/btaa792)) included in **PPanGGOLiN** predicts, for each genome, Regions of Genome Plasticity (RGPs) that are clusters of genes made of shell and cloud genomes in the pangenome graph. -Most of them arise from Horizontal gene transfer (HGT) and correspond to Genomic Islands (GIs). -RGPs from different genomes are next grouped in spots of insertion based on their conserved flanking persistent genes. - - -Those RGPs can be further divided in conserved modules by panModule ([Bazin et al. 2021](https://doi.org/10.1101/2021.12.06.471380)). Those conserved modules correspond to groups of cooccurring and colocalized genes that are gained or lost together in the variable regions of the pangenome. - - +--- <!-- [//]: # (```{toctree}) diff --git a/pyproject.toml b/pyproject.toml index c224596b..3b552bbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ doc = [ "myst-parser==2", "docutils==0.18.1", "sphinxcontrib.mermaid==0.9.2", + "linkify-it-py>=2.0.3", ] test = [ "pytest==7", diff --git a/tests/unit_tests/test_readme.py b/tests/unit_tests/test_readme.py new file mode 100644 index 00000000..9f7eaf0d --- /dev/null +++ b/tests/unit_tests/test_readme.py @@ -0,0 +1,94 @@ +"""Tests for the README rendering on both GitHub and the Sphinx documentation.""" + +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +README = ROOT / "README.md" + +_spec = importlib.util.spec_from_file_location( + "github_readme", ROOT / "docs" / "_ext" / "github_readme.py" +) +github_readme = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(github_readme) + + +def test_readme_has_no_myst_only_syntax(): + """README.md is rendered by GitHub, so it must only use GitHub-flavored Markdown.""" + issues = github_readme.find_myst_only_syntax(README.read_text(encoding="utf-8")) + assert not issues, "README.md uses MyST-only syntax:\n" + "\n".join( + f" line {line}: {issue}" for line, issue in issues + ) + + +@pytest.mark.parametrize("kind", ["NOTE", "TIP", "IMPORTANT", "WARNING", "CAUTION"]) +def test_github_alert_to_myst(kind): + text = f"Before\n\n> [!{kind}]\n> First line\n>\n> Second line\n\nAfter\n" + expected = ( + f"Before\n\n:::{{{kind.lower()}}}\nFirst line\n\nSecond line\n:::\n\nAfter\n" + ) + assert github_readme.github_alerts_to_myst(text) == expected + + +def test_github_alert_keeps_code_block(): + text = "> [!TIP]\n> Run:\n> ```shell\n> panorama -h\n> ```\n" + expected = ":::{tip}\nRun:\n```shell\npanorama -h\n```\n:::\n" + assert github_readme.github_alerts_to_myst(text) == expected + + +def test_regular_blockquote_untouched(): + text = "> Just a quote\n> on two lines\n" + assert github_readme.github_alerts_to_myst(text) == text + + +@pytest.mark.parametrize( + "line", + [ + "```{note}", + ":::{warning}", + ":::", + "See {ref}`user/install:Installation`", + "(my-label)=", + "% a MyST comment", + "+++", + ], +) +def test_myst_only_syntax_detected(line): + issues = github_readme.find_myst_only_syntax(f"Intro\n\n{line}\n") + assert [number for number, _ in issues] == [3] + + +@pytest.mark.parametrize( + "text", + [ + "```{include} ../README.md\n```\n", + "```{include} ../README.md\n:relative-images:\n```\n", + "::::{include} ../README.md\n:start-line: 2\n::::\n", + ], +) +def test_myst_include_matched(text): + match = github_readme.MYST_INCLUDE.search(text) + assert match is not None and match["path"].strip() == "../README.md" + + +def test_rebase_images(): + text = '![logo](docs/_static/logo.png) ![badge](https://x.org/b.svg) ![t](img.png "title")' + expected = '![logo](_static/logo.png) ![badge](https://x.org/b.svg) ![t](../img.png "title")' + assert ( + github_readme.rebase_images(text, Path("/repo"), Path("/repo/docs")) == expected + ) + + +def test_rebase_html_images(): + text = '<p align="center">\n <img src="docs/_static/logo.png" alt="logo">\n</p>\n<img alt="b" src=\'https://x.org/b.svg\'>' + expected = '<p align="center">\n <img src="_static/logo.png" alt="logo">\n</p>\n<img alt="b" src=\'https://x.org/b.svg\'>' + assert ( + github_readme.rebase_images(text, Path("/repo"), Path("/repo/docs")) == expected + ) + + +def test_myst_syntax_inside_code_block_ignored(): + text = "````markdown\n```{note}\nText\n```\n{ref}`a`\n````\n\nUse `{ref}` roles in the docs.\n" + assert github_readme.find_myst_only_syntax(text) == []