Code coverage

.github/workflows/pr.yml

@@ -25,7 +25,7 @@ env:
 jobs:
   tests:
     name: "Tests"
-    timeout-minutes: 30
+    timeout-minutes: 60
     runs-on: ubuntu-latest
     strategy:
       fail-fast: true
@@ -43,6 +43,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+      with:
+        # diff-cover needs more history to compute coverage diffs
+        fetch-depth: 10
 
     - name: Set up Python ${{ env.PYTHON_VERSION }}
       uses: actions/setup-python@v5
@@ -63,14 +66,25 @@ jobs:
         key: "test-datasets"
 
     - name: PyTest
-      run: invoke test.pytest
+      run: invoke test.pytest --coverage
 
     - name: Doctest
-      run: invoke test.doctest
+      run: invoke test.doctest --coverage
 
     - name: Check Notebooks
       run: invoke test.nb
 
+    - name: Combine Test Coverages & Generate XML Report
+      run: |
+        invoke test.cov-combine
+        coverage xml -i
+
+    - name: Coveralls
+      uses: coverallsapp/github-action@v2
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        files: coverage.xml
+
   lint:
     name: "Code Style"
     timeout-minutes: 10

.github/workflows/release.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   test:
     name: Tests
-    timeout-minutes: 60
+    timeout-minutes: 120
     runs-on: ${{ matrix.operating-system }}
     strategy:
       fail-fast: true

.gitignore

@@ -45,6 +45,8 @@ htmlcov/
 .nox/
 .coverage
 .coverage.*
+.coverage_*
+diff-cover.md
 .cache
 nosetests.xml
 coverage.xml

README.md

@@ -6,6 +6,7 @@
 [![Join the Discord](https://img.shields.io/discord/1235780483845984367?label=Discord)](https://discord.gg/spd2gQJGAb)
 [![Documentation](https://img.shields.io/badge/docs-latest-blue)](https://capymoa.org)
 [![GitHub](https://img.shields.io/github/stars/adaptive-machine-learning/CapyMOA?style=social)](https://github.com/adaptive-machine-learning/CapyMOA)
+[![Coverage Status](https://coveralls.io/repos/github/adaptive-machine-learning/CapyMOA/badge.svg?branch=code-cov)](https://coveralls.io/github/adaptive-machine-learning/CapyMOA?branch=code-cov)
 
 
 Machine learning library tailored for data streams. Featuring a Python API

docs/contributing/tests.md

@@ -84,3 +84,26 @@ NB_FAST=true pytest --nbmake notebooks/my_notebook.ipynb
 ```
 
 For more about `NB_FAST` read the [notebooks documentation](../docs.rst#notebooks).
+
+## Code Coverage
+
+Code coverage measures how many statements of code is executed while running
+tests. It identifies unused and untested code. We encourage contributors to
+use it to write more robust programs, but don't have a target percantage. 
+
+To generate code coverage reports add `--cov=capymoa` and `--cov-report=html` to
+the pytest command:
+
+```bash
+pytest --cov=capymoa --cov-report=html
+```
+
+Alternatively, CapyMOA's invoke testing tasks can generate coverage reports with:
+
+```bash
+invoke test --coverage
+```
+
+See also:
+* [coverage.py](https://github.com/coveragepy/coveragepy): Tool for measuring python code coverage.
+* [pytest-cov](https://pypi.org/project/pytest-cov/): PyTest plugin to automatically collect code coverage information with coverage.py.

pyproject.toml

@@ -67,6 +67,7 @@ dev=[
     "invoke~=2.2",
     "jupyter~=1.1",
     "nbmake~=1.5",
+    "pytest-cov~=7.0",
     "pytest-subtests~=0.15",
     "pytest-xdist~=3.8",
     "pytest~=9.0",
@@ -76,19 +77,12 @@ dev=[
 ]
 
 doc=[
-    # Documentation generator
-    "sphinx==8.1.3",
-    # Theme for the documentation
-    "pydata-sphinx-theme",
-    # Allows to include Jupyter notebooks in the documentation
-    "sphinx-autobuild",
-    # Allows to include Jupyter notebooks in the documentation
-    "nbsphinx",
-    # Parses markdown files
-    "myst-parser",
-    # Adds design elements to the documentation
-    "sphinx_design",
-    "sphinxcontrib-programoutput"
+    "myst-parser",                 # Parses markdown files
+    "nbsphinx",                    # Sphinx extension to include Jupyter notebooks
+    "pydata-sphinx-theme",         # Theme for the documentation
+    "sphinx_design",               # Sphinx extension for design elements
+    "sphinx==8.1.3",               # Documentation generator
+    "sphinxcontrib-programoutput", # Output of commands in the documentation
 ]
 
 [project.urls]
@@ -199,3 +193,16 @@ upload_to_vcs_release = true
 # Configuration for the formatter
 [tool.ruff]
 target-version = "py310"
+
+
+[tool.coverage.report]
+exclude_also = [
+    'pass',
+]
+
+[tool.coverage.run]
+omit = [
+    # omit pytorch-generated files in /tmp 
+    # https://stackoverflow.com/a/79415891
+    "/tmp/*"
+] 

tasks.py

@@ -17,6 +17,12 @@
 from os import environ
 
 IS_CI = environ.get("CI", "false").lower() == "true"
+COVERAGE_DEFAULT = False
+
+
+def divider(text: str):
+    """Print a divider with text centered."""
+    print(text.center(88, "-"))
 
 
 def all_exist(files: List[str] = None, directories: List[str] = None) -> bool:
@@ -195,10 +201,13 @@ def notebooks(
     Uses nbmake https://github.com/treebeardtech/nbmake to execute the notebooks
     and check for errors.
 
+    Note that nbmake does not support code coverage.
+
     The `--overwrite` flag can be used to overwrite the notebooks with the
     executed output.
     """
     assert not (not slow and overwrite), "You cannot use `--overwrite` with `--fast`."
+    env = {"COVERAGE_FILE": ".coverage.notebooks"}
 
     # Set the environment variable to run the notebooks in fast mode.
     if not slow:
@@ -220,59 +229,89 @@ def notebooks(
         "--durations=5",  # Show the duration of each notebook
     ]
     cmd += ["-n=auto"] if parallel else []  # Should we run in parallel?
-    cmd += (
-        ["--overwrite"] if overwrite else []
-    )  # Overwrite the notebooks with the executed output
+    # Overwrite the notebooks with the executed output
+    cmd += ["--overwrite"] if overwrite else []
+
     if len(skip_notebooks) > 0:
         cmd += ["--deselect " + nb for nb in skip_notebooks]  # Skip some notebooks
 
     if k_pattern:
         cmd += [f"-k {k_pattern}"]
 
-    ctx.run(" ".join(cmd), echo=True)
+    ctx.run(" ".join(cmd), echo=True, env=env)
 
 
 @task
-def pytest(ctx: Context, parallel: bool = True):
+def pytest(ctx: Context, parallel: bool = True, coverage: bool = COVERAGE_DEFAULT):
     """Run the tests using pytest."""
+    env = {"COVERAGE_FILE": ".coverage.pytest"}
+
     cmd = [
         "python -m pytest",
         "--durations=5",  # Show the duration of each test
         "--exitfirst",  # Exit instantly on first error or failed test
-        # jpype can raise irrelevant warnings:
-        # https://github.com/jpype-project/jpype/issues/561
-        "-p no:faulthandler",
     ]
+    cmd += ["--cov"] if coverage else []
     cmd += ["-n=auto"] if parallel else []
-    ctx.run(" ".join(cmd), echo=True)
+    ctx.run(" ".join(cmd), echo=True, env=env)
 
 
 @task
-def doctest(ctx: Context, parallel: bool = True):
+def doctest(ctx: Context, parallel: bool = True, coverage: bool = COVERAGE_DEFAULT):
     """Run tests defined in docstrings using pytest."""
+    env = {"COVERAGE_FILE": ".coverage.doctest"}
     cmd = [
         "python -m pytest",
         "--doctest-modules",  # Enable doctest tests
         "--durations=5",  # Show the duration of each test
         "--exitfirst",  # Exit instantly on first error or failed test
-        # jpype can raise irrelevant warnings:
-        # https://github.com/jpype-project/jpype/issues/561
-        "-p no:faulthandler",
         "src/capymoa",  # Don't run tests in the `tests` directory
     ]
+    cmd += ["--cov"] if coverage else []
     cmd += ["-n=auto"] if parallel else []
+    ctx.run(" ".join(cmd), echo=True, env=env)
+
+
+@task(aliases=["cov-combine"])
+def coverage_combine(ctx: Context):
+    """Combine coverage data from different sources."""
+    cmd = ["python -m coverage combine --keep"]
+    covfiles = [
+        ".coverage.pytest",
+        ".coverage.doctest",
+    ]
+    for covfile in covfiles:
+        if Path(covfile).exists():
+            cmd += [covfile]
     ctx.run(" ".join(cmd), echo=True)
 
 
+@task(aliases=["cov-report"], pre=[coverage_combine])
+def coverage_report(ctx: Context):
+    """Generate coverage report."""
+    ctx.run("python -m coverage html -i", echo=True)
+
+
+@task(aliases=["cov-clean"])
+def coverage_clean(ctx: Context):
+    """Clean coverage data."""
+    ctx.run("python -m coverage erase", echo=True)
+    ctx.run("rm -rf htmlcov", echo=True)
+
+
 @task
-def all_tests(ctx: Context, parallel: bool = True):
+def all_tests(ctx: Context, parallel: bool = True, coverage: bool = COVERAGE_DEFAULT):
     """Run all the tests."""
-    print("Running all pytest tests ...")
-    pytest(ctx, parallel)
-    print("Running all doctests ...")
-    doctest(ctx, parallel)
-    print("Running all notebooks ...")
+    divider("test.pytest")
+    pytest(ctx, parallel, coverage)
+    divider("test.doctest")
+    doctest(ctx, parallel, coverage)
+    divider("test.notebooks")
     notebooks(ctx, parallel)
+    if coverage:
+        divider("test.cov-report")
+        coverage_combine(ctx)
+        coverage_report(ctx)
 
 
 @task
@@ -317,6 +356,9 @@ def format(ctx: Context):
 test.add_task(notebooks, "nb")
 test.add_task(pytest, "pytest")
 test.add_task(doctest, "doctest")
+test.add_task(coverage_combine)
+test.add_task(coverage_clean)
+test.add_task(coverage_report)
 
 ns = Collection()
 ns.add_collection(docs)