Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 169 additions & 0 deletions docs/_ext/github_readme.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
"""Sphinx extension to reuse the GitHub README as the documentation landing page.

README.md is written for GitHub. When it is pulled into the docs with MyST's ``{include}``
directive, this extension replaces the directive by the README content and:

- rewrites GitHub alerts (``> [!NOTE]``) into MyST admonitions;
- warns about MyST-only syntax, which GitHub would display as raw text.

The other GitHub-flavored features (bare URL autolinks, strikethrough, task lists, mermaid
fences) are handled by MyST extensions enabled in ``conf.py``.

This module must stay importable without Sphinx so the README can be checked by the test suite.
"""

import os
import re
from pathlib import Path

README_NAME = "README.md"

GITHUB_ALERT = re.compile(
r"^> \[!(?P<kind>NOTE|TIP|IMPORTANT|WARNING|CAUTION)\][ \t]*\n(?P<body>(?:^>.*(?:\n|$))*)",
flags=re.MULTILINE,
)

FENCE = re.compile(r"^ {0,3}(?P<marker>`{3,}|~{3,}|:{3,})(?P<info>.*)$")
# Scanned left to right, so a role written inside an inline code span is consumed as code
MYST_ROLE_OR_CODE = re.compile(r"(?P<role>\{[\w:.+-]+\}`[^`]+`)|`+[^`]*`+")
MYST_INLINE_SYNTAX = {
"MyST target": re.compile(r"^\s*\([^()\s]+\)=\s*$"),
"MyST comment": re.compile(r"^\s*%"),
"MyST block break": re.compile(r"^\s*\+\+\+"),
}

MYST_INCLUDE = re.compile(
r"^(?P<fence>`{3,}|:{3,})\{include\}(?P<path>[^\n]+)\n(?P<options>(?::[^\n]*\n)*)(?P=fence)[ \t]*$",
flags=re.MULTILINE,
)
MARKDOWN_IMAGE = re.compile(
r"(?P<alt>!\[[^\]]*\])\((?P<target>[^)\s]+)(?P<title>[^)]*)\)"
)
HTML_IMAGE = re.compile(
r"(?P<before><img\b[^>]*?\bsrc=)(?P<quote>[\"'])(?P<target>[^\"']+)(?P=quote)",
flags=re.IGNORECASE,
)


def github_alerts_to_myst(text: str) -> str:
"""Convert GitHub alert blockquotes into MyST colon-fence admonitions.

Args:
text: Markdown content using GitHub alert syntax.

Returns:
The same content with every alert replaced by a MyST admonition.
"""

def _replace(match: re.Match) -> str:
body = re.sub(r"^> ?", "", match["body"], flags=re.MULTILINE).rstrip("\n")
return f":::{{{match['kind'].lower()}}}\n{body}\n:::\n"

return GITHUB_ALERT.sub(_replace, text)


def find_myst_only_syntax(text: str) -> list[tuple[int, str]]:
"""Find MyST syntax that GitHub cannot render.

Content of regular code blocks is ignored, so documenting MyST syntax inside a code block is fine.

Args:
text: Markdown content meant to be rendered by GitHub.

Returns:
A list of ``(line_number, description)`` tuples, one per offending line (1-indexed).
"""
issues = []
open_fence = None
for line_number, line in enumerate(text.splitlines(), start=1):
fence = FENCE.match(line)
if open_fence is not None:
if (
fence
and fence["marker"][0] == open_fence[0]
and len(fence["marker"]) >= len(open_fence)
):
if not fence["info"].strip():
open_fence = None
continue
if fence:
marker, info = fence["marker"], fence["info"].strip()
if marker.startswith(":"):
issues.append((line_number, f"MyST colon fence '{line.strip()}'"))
elif info.startswith("{"):
issues.append((line_number, f"MyST directive '{line.strip()}'"))
open_fence = marker
continue
if any(match["role"] for match in MYST_ROLE_OR_CODE.finditer(line)):
issues.append((line_number, f"MyST role in '{line.strip()}'"))
for name, pattern in MYST_INLINE_SYNTAX.items():
if pattern.match(line):
issues.append((line_number, f"{name} '{line.strip()}'"))
return issues


def rebase_images(text: str, readme_dir: Path, doc_dir: Path) -> str:
"""Make relative image paths of the README relative to the including document.

Reproduces the ``:relative-images:`` option of MyST's ``{include}`` directive, which only
handles Markdown images, and also applies it to the ``src`` of HTML ``<img>`` tags. Raw HTML is
copied as is to the output, so an HTML image only shows up if its file is part of the output
too (for example, in ``html_static_path``).

Args:
text: README content.
readme_dir: Directory containing the README.
doc_dir: Directory containing the document that includes the README.

Returns:
The content with every relative image path rebased on ``doc_dir``.
"""

def _rebase(target: str) -> str:
if re.match(r"^([a-z][a-z0-9+.-]*:|/|#)", target, flags=re.IGNORECASE):
return target
return Path(os.path.relpath(readme_dir / target, doc_dir)).as_posix()

text = MARKDOWN_IMAGE.sub(
lambda m: f"{m['alt']}({_rebase(m['target'])}{m['title']})", text
)
return HTML_IMAGE.sub(
lambda m: f"{m['before']}{m['quote']}{_rebase(m['target'])}{m['quote']}", text
)


def expand_readme_include(app, docname: str, source: list[str]) -> None:
"""Sphinx ``source-read`` handler inlining the README converted from GitHub to MyST syntax.

MyST's ``{include}`` directive reads the file itself, so the README cannot be converted when it
is included. The directive is instead replaced by the converted README content before parsing.
"""
from sphinx.util import logging

logger = logging.getLogger(__name__)
doc_path = Path(app.env.doc2path(docname))

def _replace(match: re.Match) -> str:
readme_path = (doc_path.parent / match["path"].strip()).resolve()
if readme_path.name != README_NAME:
return match[0]
app.env.note_dependency(str(readme_path))
text = readme_path.read_text(encoding="utf-8")
for line_number, issue in find_myst_only_syntax(text):
logger.warning(
"%s will not render on GitHub, use GitHub-flavored Markdown instead",
issue,
location=f"{readme_path}:{line_number}",
)
text = github_alerts_to_myst(text)
if re.search(r"^:relative-images:", match["options"], flags=re.MULTILINE):
text = rebase_images(text, readme_path.parent, doc_path.parent)
return text

source[0] = MYST_INCLUDE.sub(_replace, source[0])


def setup(app):
"""Register the extension in Sphinx."""
app.connect("source-read", expand_readme_include)
return {"parallel_read_safe": True, "parallel_write_safe": True}
10 changes: 9 additions & 1 deletion docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent / "_ext"))


def generate_command_reference(_app):
script = (
Expand All @@ -38,7 +40,6 @@ def setup(app):
open(Path(__file__).resolve().parents[1] / "VERSION").read().rstrip()
) # Get release number in the VERSION file


# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
Expand All @@ -52,8 +53,15 @@ def setup(app):
"sphinx.ext.autodoc",
"sphinx_search.extension",
"sphinxcontrib.mermaid",
"github_readme", # Local extension (_ext/): renders the GitHub README as the landing page
]

# These extensions mirror GitHub-flavored Markdown so README.md renders the same in both places
myst_enable_extensions = ["linkify", "strikethrough", "tasklist"]
myst_fence_as_directive = ["mermaid"]
myst_linkify_fuzzy_links = (
False # Like GitHub, only autolink http(s):// and www. URLs, not "README.md"
)

source_suffix = {".md": "markdown"}

Expand Down
43 changes: 3 additions & 40 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,48 +3,11 @@
% You can adapt this file completely to your liking, but it should at least
% contain the root `toctree` directive.


# PPanGGOLiN: Depicting microbial species diversity via a Partitioned PanGenome Graph Of Linked Neighbors


[![Actions](https://img.shields.io/github/actions/workflow/status/labgem/PPanGGOLiN/main.yml?branch=master&label=CI&logo=github)](https://github.com/labgem/PPanGGOLiN/actions/workflows/main.yml)
[![License](https://anaconda.org/bioconda/ppanggolin/badges/license.svg)](http://www.cecill.info/licences.fr.html)
[![Bioconda](https://img.shields.io/conda/vn/bioconda/ppanggolin?style=flat-square&maxAge=3600&logo=anaconda)](https://anaconda.org/bioconda/ppanggolin)
[![PyPI version](https://badge.fury.io/py/PPanGGOLiN.svg?cache-control=no-cache)](https://pypi.org/project/PPanGGOLiN/)
[![Source](https://img.shields.io/badge/source-GitHub-303030.svg?maxAge=2678400&style=flat-square)](https://github.com/labgem/ppanggolin/)
[![GitHub issues](https://img.shields.io/github/issues/labgem/ppanggolin.svg?style=flat-square&maxAge=600)](https://github.com/labgem/ppanggolin/issues)
[![Docs](https://img.shields.io/readthedocs/ppanggolin/latest?style=flat-square&maxAge=600)](https://ppanggolin.readthedocs.io)
[![Downloads](https://anaconda.org/bioconda/ppanggolin/badges/downloads.svg)](https://bioconda.github.io/recipes/ppanggolin/README.html#download-stats)


```{image} _static/logo.png
:alt: ppangolin logo
:align: center
:width: 437
```{include} ../README.md
:relative-images:
```

<br>


**PPanGGOLiN**
([Gautreau et al. 2020](https://doi.org/10.1371/journal.pcbi.1007732)) is a software suite used to create and manipulate prokaryotic pangenomes from a set of either assembled genomic DNA sequences or provided genome annotations.
It is designed to scale up to tens of thousands of genomes.
It has the specificity to partition the pangenome using a statistical approach rather than using fixed thresholds which gives it the ability to work with low-quality data such as *Metagenomic Assembled Genomes (MAGs)* or *Single-cell Amplified Genomes (SAGs)* thus taking advantage of large scale environmental studies and letting users study the pangenome of uncultivable species.

**PPanGGOLiN** builds pangenomes through a graphical model and a statistical method to partition gene families in persistent, shell and cloud genomes.
It integrates both information on protein-coding genes and their genomic neighborhood to build a graph of gene families where each node is a gene family, and each edge is a relation of genetic contiguity.
The partitioning method promotes that two gene families that are consistent neighbors in the graph are more likely to belong to the same partition.
It results in a Partitioned Pangenome Graph (PPG) made of persistent, shell and cloud nodes drawing genomes on rails like a subway map to help biologists navigate the great diversity of microbial life.


Moreover, the panRGP method ([Bazin et al. 2020](https://doi.org/10.1093/bioinformatics/btaa792)) included in **PPanGGOLiN** predicts, for each genome, Regions of Genome Plasticity (RGPs) that are clusters of genes made of shell and cloud genomes in the pangenome graph.
Most of them arise from Horizontal gene transfer (HGT) and correspond to Genomic Islands (GIs).
RGPs from different genomes are next grouped in spots of insertion based on their conserved flanking persistent genes.


Those RGPs can be further divided in conserved modules by panModule ([Bazin et al. 2021](https://doi.org/10.1101/2021.12.06.471380)). Those conserved modules correspond to groups of cooccurring and colocalized genes that are gained or lost together in the variable regions of the pangenome.


---
<!--
[//]: # (```{toctree})

Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ doc = [
"myst-parser==2",
"docutils==0.18.1",
"sphinxcontrib.mermaid==0.9.2",
"linkify-it-py>=2.0.3",
]
test = [
"pytest==7",
Expand Down
94 changes: 94 additions & 0 deletions tests/unit_tests/test_readme.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
"""Tests for the README rendering on both GitHub and the Sphinx documentation."""

import importlib.util
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parents[2]
README = ROOT / "README.md"

_spec = importlib.util.spec_from_file_location(
"github_readme", ROOT / "docs" / "_ext" / "github_readme.py"
)
github_readme = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(github_readme)


def test_readme_has_no_myst_only_syntax():
"""README.md is rendered by GitHub, so it must only use GitHub-flavored Markdown."""
issues = github_readme.find_myst_only_syntax(README.read_text(encoding="utf-8"))
assert not issues, "README.md uses MyST-only syntax:\n" + "\n".join(
f" line {line}: {issue}" for line, issue in issues
)


@pytest.mark.parametrize("kind", ["NOTE", "TIP", "IMPORTANT", "WARNING", "CAUTION"])
def test_github_alert_to_myst(kind):
text = f"Before\n\n> [!{kind}]\n> First line\n>\n> Second line\n\nAfter\n"
expected = (
f"Before\n\n:::{{{kind.lower()}}}\nFirst line\n\nSecond line\n:::\n\nAfter\n"
)
assert github_readme.github_alerts_to_myst(text) == expected


def test_github_alert_keeps_code_block():
text = "> [!TIP]\n> Run:\n> ```shell\n> panorama -h\n> ```\n"
expected = ":::{tip}\nRun:\n```shell\npanorama -h\n```\n:::\n"
assert github_readme.github_alerts_to_myst(text) == expected


def test_regular_blockquote_untouched():
text = "> Just a quote\n> on two lines\n"
assert github_readme.github_alerts_to_myst(text) == text


@pytest.mark.parametrize(
"line",
[
"```{note}",
":::{warning}",
":::",
"See {ref}`user/install:Installation`",
"(my-label)=",
"% a MyST comment",
"+++",
],
)
def test_myst_only_syntax_detected(line):
issues = github_readme.find_myst_only_syntax(f"Intro\n\n{line}\n")
assert [number for number, _ in issues] == [3]


@pytest.mark.parametrize(
"text",
[
"```{include} ../README.md\n```\n",
"```{include} ../README.md\n:relative-images:\n```\n",
"::::{include} ../README.md\n:start-line: 2\n::::\n",
],
)
def test_myst_include_matched(text):
match = github_readme.MYST_INCLUDE.search(text)
assert match is not None and match["path"].strip() == "../README.md"


def test_rebase_images():
text = '![logo](docs/_static/logo.png) ![badge](https://x.org/b.svg) ![t](img.png "title")'
expected = '![logo](_static/logo.png) ![badge](https://x.org/b.svg) ![t](../img.png "title")'
assert (
github_readme.rebase_images(text, Path("/repo"), Path("/repo/docs")) == expected
)


def test_rebase_html_images():
text = '<p align="center">\n <img src="docs/_static/logo.png" alt="logo">\n</p>\n<img alt="b" src=\'https://x.org/b.svg\'>'
expected = '<p align="center">\n <img src="_static/logo.png" alt="logo">\n</p>\n<img alt="b" src=\'https://x.org/b.svg\'>'
assert (
github_readme.rebase_images(text, Path("/repo"), Path("/repo/docs")) == expected
)


def test_myst_syntax_inside_code_block_ignored():
text = "````markdown\n```{note}\nText\n```\n{ref}`a`\n````\n\nUse `{ref}` roles in the docs.\n"
assert github_readme.find_myst_only_syntax(text) == []
Loading