ci: add code coverage

Author: tachyonicClock

.github/workflows/pr.yml

@@ -43,6 +43,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+      with:
+        # diff-cover needs more history to compute coverage diffs
+        fetch-depth: 10
 
     - name: Set up Python ${{ env.PYTHON_VERSION }}
       uses: actions/setup-python@v5
@@ -63,13 +66,21 @@ jobs:
         key: "test-datasets"
 
     - name: PyTest
-      run: invoke test.pytest
+      run: invoke test.pytest --coverage
 
     - name: Doctest
-      run: invoke test.doctest
+      run: invoke test.doctest --coverage
 
     - name: Check Notebooks
-      run: invoke test.nb
+      run: invoke test.nb --coverage
+
+    - name: Combine Test Coverages
+      run: invoke test.cov-combine
+        
+    - name: Coveralls
+      uses: coverallsapp/github-action@v2
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
 
   lint:
     name: "Code Style"

.gitignore

@@ -45,6 +45,8 @@ htmlcov/
 .nox/
 .coverage
 .coverage.*
+.coverage_*
+diff-cover.md
 .cache
 nosetests.xml
 coverage.xml

docs/contributing/tests.md

@@ -84,3 +84,27 @@ NB_FAST=true pytest --nbmake notebooks/my_notebook.ipynb
 ```
 
 For more about `NB_FAST` read the [notebooks documentation](../docs.rst#notebooks).
+
+## Code Coverage
+
+Code coverage measures how many statements of code is executed while running
+tests. It identifies unused and untested code. We encourage contributors to
+use it to write more robust programs, but don't have a target percantage. 
+
+To generate code coverage reports add `--cov=capymoa` and `--cov-report=html` to
+the pytest command:
+
+```bash
+pytest --cov=capymoa --cov-report=html
+```
+
+Alternatively, CapyMOA's invoke testing tasks can generate coverage reports with:
+
+```bash
+invoke test --coverage
+```
+
+See also:
+* [coverage.py](https://github.com/coveragepy/coveragepy): Tool for measuring python code coverage.
+* [pytest-cov](https://pypi.org/project/pytest-cov/): PyTest plugin to automatically collect code coverage information with coverage.py.
+* [diff-cover](https://github.com/Bachmann1234/diff-cover): Program to generate a coverage report for only lines changed between branches.

pyproject.toml

@@ -67,6 +67,7 @@ dev=[
     "invoke~=2.2",
     "jupyter~=1.1",
     "nbmake~=1.5",
+    "pytest-cov~=7.0",
     "pytest-subtests~=0.15",
     "pytest-xdist~=3.8",
     "pytest~=9.0",
@@ -76,19 +77,12 @@ dev=[
 ]
 
 doc=[
-    # Documentation generator
-    "sphinx==8.1.3",
-    # Theme for the documentation
-    "pydata-sphinx-theme",
-    # Allows to include Jupyter notebooks in the documentation
-    "sphinx-autobuild",
-    # Allows to include Jupyter notebooks in the documentation
-    "nbsphinx",
-    # Parses markdown files
-    "myst-parser",
-    # Adds design elements to the documentation
-    "sphinx_design",
-    "sphinxcontrib-programoutput"
+    "myst-parser",                 # Parses markdown files
+    "nbsphinx",                    # Sphinx extension to include Jupyter notebooks
+    "pydata-sphinx-theme",         # Theme for the documentation
+    "sphinx_design",               # Sphinx extension for design elements
+    "sphinx==8.1.3",               # Documentation generator
+    "sphinxcontrib-programoutput", # Output of commands in the documentation
 ]
 
 [project.urls]

src/capymoa/misc.py

@@ -116,3 +116,7 @@ def save_stream_arff(file: TextIO | Path | str, stream: Stream) -> None:
         file.write(str(moa_header))
         for instance in tqdm.tqdm(stream, desc="Saving to ARFF"):
             file.write(str(instance.java_instance.toString()) + "\n")
+
+
+def _TODO_REMOVE_ME_I_AM_A_TEST():
+    print("This is a test function that should be removed.")

tasks.py

@@ -17,6 +17,12 @@
 from os import environ
 
 IS_CI = environ.get("CI", "false").lower() == "true"
+COVERAGE_DEFAULT = False
+
+
+def divider(text: str):
+    """Print a divider with text centered."""
+    print(text.center(88, "-"))
 
 
 def all_exist(files: List[str] = None, directories: List[str] = None) -> bool:
@@ -189,6 +195,7 @@ def notebooks(
     k_pattern: Optional[str] = None,
     slow: bool = False,
     no_skip: bool = False,
+    coverage: bool = COVERAGE_DEFAULT,
 ):
     """Run the notebooks and check for errors.
 
@@ -199,6 +206,7 @@ def notebooks(
     executed output.
     """
     assert not (not slow and overwrite), "You cannot use `--overwrite` with `--fast`."
+    env = {"COVERAGE_FILE": ".coverage_notebooks"}
 
     # Set the environment variable to run the notebooks in fast mode.
     if not slow:
@@ -220,59 +228,95 @@ def notebooks(
         "--durations=5",  # Show the duration of each notebook
     ]
     cmd += ["-n=auto"] if parallel else []  # Should we run in parallel?
-    cmd += (
-        ["--overwrite"] if overwrite else []
-    )  # Overwrite the notebooks with the executed output
+    # Overwrite the notebooks with the executed output
+    cmd += ["--overwrite"] if overwrite else []
+    cmd += ["--cov=capymoa", ""] if coverage else []
+
     if len(skip_notebooks) > 0:
         cmd += ["--deselect " + nb for nb in skip_notebooks]  # Skip some notebooks
 
     if k_pattern:
         cmd += [f"-k {k_pattern}"]
 
-    ctx.run(" ".join(cmd), echo=True)
+    ctx.run(" ".join(cmd), echo=True, env=env)
 
 
 @task
-def pytest(ctx: Context, parallel: bool = True):
+def pytest(ctx: Context, parallel: bool = True, coverage: bool = COVERAGE_DEFAULT):
     """Run the tests using pytest."""
+    env = {"COVERAGE_FILE": ".coverage_pytest"}
+
     cmd = [
         "python -m pytest",
         "--durations=5",  # Show the duration of each test
         "--exitfirst",  # Exit instantly on first error or failed test
-        # jpype can raise irrelevant warnings:
-        # https://github.com/jpype-project/jpype/issues/561
-        "-p no:faulthandler",
     ]
+    if coverage:
+        cmd.extend(
+            [
+                "--cov=capymoa",
+            ]
+        )
     cmd += ["-n=auto"] if parallel else []
-    ctx.run(" ".join(cmd), echo=True)
+    ctx.run(" ".join(cmd), echo=True, env=env)
 
 
 @task
-def doctest(ctx: Context, parallel: bool = True):
+def doctest(ctx: Context, parallel: bool = True, coverage: bool = COVERAGE_DEFAULT):
     """Run tests defined in docstrings using pytest."""
+    env = {"COVERAGE_FILE": ".coverage_doctest"}
     cmd = [
         "python -m pytest",
+        "--cov=capymoa" if coverage else "",
         "--doctest-modules",  # Enable doctest tests
         "--durations=5",  # Show the duration of each test
         "--exitfirst",  # Exit instantly on first error or failed test
-        # jpype can raise irrelevant warnings:
-        # https://github.com/jpype-project/jpype/issues/561
-        "-p no:faulthandler",
         "src/capymoa",  # Don't run tests in the `tests` directory
     ]
     cmd += ["-n=auto"] if parallel else []
+    ctx.run(" ".join(cmd), echo=True, env=env)
+
+
+@task(aliases=["cov-combine"])
+def coverage_combine(ctx: Context):
+    """Combine coverage data from different sources."""
+    cmd = ["python -m coverage combine --keep"]
+    covfiles = [
+        ".coverage_pytest",
+        ".coverage_doctest",
+        ".coverage_notebooks",
+    ]
+    for covfile in covfiles:
+        if Path(covfile).exists():
+            cmd += [covfile]
     ctx.run(" ".join(cmd), echo=True)
 
 
+@task(aliases=["cov-report"], pre=[coverage_combine])
+def coverage_report(ctx: Context):
+    """Generate coverage report."""
+    ctx.run("python -m coverage html", echo=True)
+
+
+@task(aliases=["cov-clean"])
+def coverage_clean(ctx: Context):
+    """Clean coverage data."""
+    ctx.run("python -m coverage erase", echo=True)
+    ctx.run("rm -rf htmlcov", echo=True)
+
+
 @task
-def all_tests(ctx: Context, parallel: bool = True):
+def all_tests(ctx: Context, parallel: bool = True, coverage: bool = COVERAGE_DEFAULT):
     """Run all the tests."""
-    print("Running all pytest tests ...")
-    pytest(ctx, parallel)
-    print("Running all doctests ...")
-    doctest(ctx, parallel)
-    print("Running all notebooks ...")
+    divider("test.pytest")
+    pytest(ctx, parallel, coverage)
+    divider("test.doctest")
+    doctest(ctx, parallel, coverage)
+    divider("test.notebooks")
     notebooks(ctx, parallel)
+    if coverage:
+        divider("test.cov-report")
+        coverage_combine(ctx)
 
 
 @task
@@ -317,6 +361,9 @@ def format(ctx: Context):
 test.add_task(notebooks, "nb")
 test.add_task(pytest, "pytest")
 test.add_task(doctest, "doctest")
+test.add_task(coverage_combine)
+test.add_task(coverage_clean)
+test.add_task(coverage_report)
 
 ns = Collection()
 ns.add_collection(docs)