diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 886b0fe..0000000 --- a/.flake8 +++ /dev/null @@ -1,31 +0,0 @@ -[flake8] -max-line-length = 88 -exclude= - .git, - venv, - docs/conf.py, -ignore= - # False positive for white space before ':' on list slice - # black should format these correctly - E203, - # Block comment should start with '# ' - # Not if it's a commented out line - E265, - # Ambiguous variable names - # It's absolutely fine to have i and I - E741, - # List comprehension redefines variable - # Re-using throw-away variables like `i`, `x`, etc. is a Good Idea - F812, - # Blank line at end of file - # This increases readability - W391, - # Line break before binary operator - # This is now actually advised in pep8 - W503, - # Line break after binary operator - W504, - # Invalid escape sequence - # These happen all the time in latex parts of docstrings, - # e.g. \sigma - W605, \ No newline at end of file diff --git a/.gitignore b/.gitignore index efa8497..8c80c2e 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.vscode # Spyder project settings .spyderproject diff --git a/docs/conf.py b/docs/conf.py index cdce3a1..a100001 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,6 +31,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + "sphinx.ext.doctest", ] # Add any paths that contain templates here, relative to this directory. @@ -48,8 +49,3 @@ # a list of builtin themes. # html_theme = 'alabaster' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] diff --git a/docs/examples/docstring_doi.py b/docs/examples/docstring_doi_reference.py similarity index 100% rename from docs/examples/docstring_doi.py rename to docs/examples/docstring_doi_reference.py diff --git a/docs/examples/example.ipynb b/docs/examples/example.ipynb new file mode 100644 index 0000000..c11e036 --- /dev/null +++ b/docs/examples/example.ipynb @@ -0,0 +1,404 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using R2T2 in a script" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This example shows how to annotate functions and obtain the bibliography from the script. For running R2T2 from command line, see our [documentation](https://r2t2.readthedocs.io/en/latest/#)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic usage" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "R2T2 works by decorating functions, classes or methods where particular algorithms described in a paper are implemented or data stored in a repository is used. General execution of code silently passes these decorators, but remembers how and where they were called. The decorators include a short description of the thing being reference, and the reference itself in any sensible format." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from r2t2 import add_reference\n", + "\n", + "@add_reference(short_purpose=\"Original implementation of R2T2\",\n", + " reference=\"Diego Alonso-Álvarez, et al.\"\n", + " \"(2018, February 27). Solcore (Version 5.1.0). Zenodo.\"\n", + " \"http://doi.org/10.5281/zenodo.1185316\")\n", + "def my_great_function():\n", + " pass" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When tracking is on, the function will be added to the `BIBLIOGRAPHY` object whenever the function is called:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Referenced in: my_great_function\n", + "Source file: \n", + "Line: 3\n", + "\t[1] Original implementation of R2T2 - Diego Alonso-Álvarez, et al.(2018, February 27). Solcore (Version 5.1.0). Zenodo.http://doi.org/10.5281/zenodo.1185316\n", + "\n", + "\n" + ] + } + ], + "source": [ + "from r2t2 import BIBLIOGRAPHY\n", + "\n", + "BIBLIOGRAPHY.tracking()\n", + "my_great_function()\n", + "print(BIBLIOGRAPHY)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tracking several functions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If several functions are annotated, they will all be tracked by the bibliography (once tracking is enabled). This also works for classes, and their methods. Note that in a notebook, the class's `__init__` method, rather than the class itself, should be decorated" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: class reference does not work in a jupyter notebook. See #41\n", + "class MyGreatClass():\n", + " @add_reference(short_purpose=\"Demonstrate class reference\",\n", + " reference=\"A reference for a class\")\n", + " def __init__(self):\n", + " pass\n", + " \n", + " @add_reference(short_purpose=\"Demonstrate class method reference\",\n", + " reference=\"A reference for a class method\")\n", + " def my_great_method(self):\n", + " pass" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We clear the bibliography so that it starts off empty" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "BIBLIOGRAPHY.clear()\n", + "print(BIBLIOGRAPHY)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can add the class" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Referenced in: __init__\n", + "Source file: \n", + "Line: 3\n", + "\t[1] Demonstrate class reference - A reference for a class\n", + "\n", + "\n" + ] + } + ], + "source": [ + "my_class = MyGreatClass()\n", + "print(BIBLIOGRAPHY)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Since the method has not yet been called, its reference is not included in the bibliography. After we call the method, it gets added to the bibliography" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Referenced in: __init__\n", + "Source file: \n", + "Line: 3\n", + "\t[1] Demonstrate class reference - A reference for a class\n", + "\n", + "Referenced in: my_great_method\n", + "Source file: \n", + "Line: 8\n", + "\t[1] Demonstrate class method reference - A reference for a class method\n", + "\n", + "\n" + ] + } + ], + "source": [ + "my_class.my_great_method()\n", + "print(BIBLIOGRAPHY)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can also call our original function" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Referenced in: __init__\n", + "Source file: \n", + "Line: 3\n", + "\t[1] Demonstrate class reference - A reference for a class\n", + "\n", + "Referenced in: my_great_method\n", + "Source file: \n", + "Line: 8\n", + "\t[1] Demonstrate class method reference - A reference for a class method\n", + "\n", + "Referenced in: my_great_function\n", + "Source file: \n", + "Line: 3\n", + "\t[1] Original implementation of R2T2 - Diego Alonso-Álvarez, et al.(2018, February 27). Solcore (Version 5.1.0). Zenodo.http://doi.org/10.5281/zenodo.1185316\n", + "\n", + "\n" + ] + } + ], + "source": [ + "my_great_function()\n", + "print(BIBLIOGRAPHY)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can also add a function with two references" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [], + "source": [ + "@add_reference(short_purpose=\"some comment\", reference=\"Reference 1\")\n", + "@add_reference(short_purpose=\"another comment\", reference=\"Reference 2\")\n", + "def my_multiple_function():\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Referenced in: __init__\n", + "Source file: \n", + "Line: 3\n", + "\t[1] Demonstrate class reference - A reference for a class\n", + "\n", + "Referenced in: my_great_method\n", + "Source file: \n", + "Line: 8\n", + "\t[1] Demonstrate class method reference - A reference for a class method\n", + "\n", + "Referenced in: my_great_function\n", + "Source file: \n", + "Line: 3\n", + "\t[1] Original implementation of R2T2 - Diego Alonso-Álvarez, et al.(2018, February 27). Solcore (Version 5.1.0). Zenodo.http://doi.org/10.5281/zenodo.1185316\n", + "\n", + "Referenced in: my_multiple_function\n", + "Source file: \n", + "Line: 1\n", + "\t[1] some comment - Reference 1\n", + "\t[2] another comment - Reference 2\n", + "\n", + "\n" + ] + } + ], + "source": [ + "my_multiple_function()\n", + "print(BIBLIOGRAPHY)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Saving the bibliography" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The bibliography's string output can be saved to any file, for example" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"bibliography.txt\",\"w\") as f:\n", + " f.write(str(BIBLIOGRAPHY))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And then read again" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Referenced in: __init__\n", + "Source file: \n", + "Line: 3\n", + "\t[1] Demonstrate class reference - A reference for a class\n", + "\n", + "Referenced in: my_great_method\n", + "Source file: \n", + "Line: 8\n", + "\t[1] Demonstrate class method reference - A reference for a class method\n", + "\n", + "Referenced in: my_great_function\n", + "Source file: \n", + "Line: 3\n", + "\t[1] Original implementation of R2T2 - Diego Alonso-Álvarez, et al.(2018, February 27). Solcore (Version 5.1.0). Zenodo.http://doi.org/10.5281/zenodo.1185316\n", + "\n", + "\n" + ] + } + ], + "source": [ + "with open(\"bibliography.txt\",\"r\") as f:\n", + " print(f.read())" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.remove(\"bibliography.txt\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/examples/minimal-class.py b/docs/examples/minimal-class.py deleted file mode 100644 index 420c733..0000000 --- a/docs/examples/minimal-class.py +++ /dev/null @@ -1,9 +0,0 @@ -from r2t2 import add_reference - - -@add_reference(short_purpose="Original implementation of R2T2", - reference="Diego Alonso-Álvarez, et al." - "(2018, February 27). Solcore (Version 5.1.0). Zenodo." - "http://doi.org/10.5281/zenodo.1185316") -class MyGreatClass(): - pass diff --git a/docs/examples/minimal-method.py b/docs/examples/minimal-method.py deleted file mode 100644 index 7198662..0000000 --- a/docs/examples/minimal-method.py +++ /dev/null @@ -1,11 +0,0 @@ -from r2t2 import add_reference - - -class MyGreatClass(): - - @add_reference(short_purpose="Original implementation of R2T2", - reference="Diego Alonso-Álvarez, et al." - "(2018, February 27). Solcore (Version 5.1.0). Zenodo." - "http://doi.org/10.5281/zenodo.1185316") - def my_great_function(self): - pass diff --git a/docs/examples/minimal.py b/docs/examples/minimal.py index f579989..bb1f082 100644 --- a/docs/examples/minimal.py +++ b/docs/examples/minimal.py @@ -1,9 +1,11 @@ from r2t2 import add_reference -@add_reference(short_purpose="Original implementation of R2T2", - reference="Diego Alonso-Álvarez, et al." - "(2018, February 27). Solcore (Version 5.1.0). Zenodo." - "http://doi.org/10.5281/zenodo.1185316") +@add_reference( + short_purpose="Original implementation of R2T2", + reference="Diego Alonso-Álvarez, et al." + "(2018, February 27). Solcore (Version 5.1.0). Zenodo." + "http://doi.org/10.5281/zenodo.1185316", +) def my_great_function(): pass diff --git a/docs/examples/minimal_class.py b/docs/examples/minimal_class.py new file mode 100644 index 0000000..2761c96 --- /dev/null +++ b/docs/examples/minimal_class.py @@ -0,0 +1,11 @@ +from r2t2 import add_reference + + +@add_reference( + short_purpose="Original implementation of R2T2", + reference="Diego Alonso-Álvarez, et al." + "(2018, February 27). Solcore (Version 5.1.0). Zenodo." + "http://doi.org/10.5281/zenodo.1185316", +) +class MyGreatClass: + pass diff --git a/docs/examples/minimal_method.py b/docs/examples/minimal_method.py new file mode 100644 index 0000000..74c4704 --- /dev/null +++ b/docs/examples/minimal_method.py @@ -0,0 +1,12 @@ +from r2t2 import add_reference + + +class MyGreatClass: + @add_reference( + short_purpose="Original implementation of R2T2", + reference="Diego Alonso-Álvarez, et al." + "(2018, February 27). Solcore (Version 5.1.0). Zenodo." + "http://doi.org/10.5281/zenodo.1185316", + ) + def my_great_function(self): + pass diff --git a/docs/examples/reference_docstring.py b/docs/examples/reference_docstring.py new file mode 100644 index 0000000..b6d66fe --- /dev/null +++ b/docs/examples/reference_docstring.py @@ -0,0 +1,8 @@ +def my_great_function(): + """ + Original implementation of R2T2 + Diego Alonso-Álvarez, et al. + (2018, February 27). Solcore (Version 5.1.0). Zenodo. + http://doi.org/10.5281/zenodo.1185316 + """ + pass diff --git a/docs/user-guide/references.rst b/docs/user-guide/references.rst index 18aa2da..172d793 100644 --- a/docs/user-guide/references.rst +++ b/docs/user-guide/references.rst @@ -15,13 +15,13 @@ In Function In Class -------- -.. literalinclude:: ../examples/minimal-class.py +.. literalinclude:: ../examples/minimal_class.py :language: python In Method --------- -.. literalinclude:: ../examples/minimal-method.py +.. literalinclude:: ../examples/minimal_method.py :language: python Two or More References @@ -35,7 +35,7 @@ As Docstring R2T2 will parse the docstring searching for the DOI. -.. literalinclude:: ../examples/docstring_doi.py +.. literalinclude:: ../examples/docstring_doi_reference.py :language: python R2T2 will also search for Sphinx's ``cite`` directive. diff --git a/r2t2/core.py b/r2t2/core.py index 210857d..6f8ed98 100755 --- a/r2t2/core.py +++ b/r2t2/core.py @@ -78,7 +78,12 @@ def add_reference( @wrapt.decorator(enabled=lambda: BIBLIOGRAPHY.track_references) def wrapper(wrapped, instance, args, kwargs): source = inspect.getsourcefile(wrapped) - line = inspect.getsourcelines(wrapped)[1] + try: + line = inspect.getsourcelines(wrapped)[1] + # add exception for testing notebook in a subprocess + except OSError: + line = "n/a" + identifier = f"{source}:{line}" if identifier in BIBLIOGRAPHY and ref in BIBLIOGRAPHY[identifier].references: diff --git a/run_tests_examples_and_docs.py b/run_tests_examples_and_docs.py new file mode 100644 index 0000000..4e4f80d --- /dev/null +++ b/run_tests_examples_and_docs.py @@ -0,0 +1,211 @@ +# +# Runs all examples to make sure they don't return errors +# +# The code in this file is adapted from Pints +# (see https://github.com/pints-team/pints) +# +import re +import os +import sys +import subprocess + + +def run_notebook_and_scripts(executable="python"): + """ + Runs example scripts and Jupyter notebooks. Exits if they fail. + """ + # Scan and run + print("Testing notebooks and scripts with executable `" + str(executable) + "`") + if not scan_for_nb_and_scripts("docs/examples", True, executable): + print("\nErrors encountered in notebooks") + sys.exit(1) + print("\nOK") + + +def scan_for_nb_and_scripts(root, recursive=True, executable="python"): + """ + Scans for, and tests, all notebooks and scripts in a directory. + """ + ok = True + debug = False + + # Scan path + for filename in os.listdir(root): + path = os.path.join(root, filename) + + # Recurse into subdirectories + if recursive and os.path.isdir(path): + # Ignore hidden directories + if filename[:1] == ".": + continue + ok &= scan_for_nb_and_scripts(path, recursive, executable) + + # Test notebooks + if os.path.splitext(path)[1] == ".ipynb": + if debug: + print(path) + else: + ok &= test_notebook(path, executable) + # Test scripts + elif os.path.splitext(path)[1] == ".py": + if debug: + print(path) + else: + ok &= test_script(path, executable) + + # Return True if every notebook is ok + return ok + + +def test_notebook(path, executable="python"): + """ + Tests a single notebook, exists if it doesn't finish. + """ + import nbconvert + + print("Test " + path + " ... ", end="") + sys.stdout.flush() + + # Load notebook, convert to python + e = nbconvert.exporters.PythonExporter() + code, __ = e.from_filename(path) + + # Remove coding statement, if present + code = "\n".join([x for x in code.splitlines() if x[:9] != "# coding"]) + + # Tell matplotlib not to produce any figures + env = dict(os.environ) + env["MPLBACKEND"] = "Template" + + # If notebook makes use of magic commands then + # the script must be ran using ipython + # https://github.com/jupyter/nbconvert/issues/503#issuecomment-269527834 + executable = ( + "ipython" + if (("run_cell_magic(" in code) or ("run_line_magic(" in code)) + else executable + ) + + # Run in subprocess + cmd = [executable] + ["-c", code] + try: + p = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env + ) + stdout, stderr = p.communicate() + # TODO: Use p.communicate(timeout=3600) if Python3 only + if p.returncode != 0: + # Show failing code, output and errors before returning + print("ERROR") + print("-- script " + "-" * (79 - 10)) + for i, line in enumerate(code.splitlines()): + j = str(1 + i) + print(j + " " * (5 - len(j)) + line) + print("-- stdout " + "-" * (79 - 10)) + print(str(stdout, "utf-8")) + print("-- stderr " + "-" * (79 - 10)) + print(str(stderr, "utf-8")) + print("-" * 79) + return False + except KeyboardInterrupt: + p.terminate() + print("ABORTED") + sys.exit(1) + + # Sucessfully run + print("ok") + return True + + +def test_script(path, executable="python"): + """ + Tests a single notebook, exists if it doesn't finish. + """ + print("Test " + path + " ... ", end="") + sys.stdout.flush() + + # Tell matplotlib not to produce any figures + env = dict(os.environ) + env["MPLBACKEND"] = "Template" + + # Run in subprocess + cmd = [executable] + [path] + try: + p = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env + ) + stdout, stderr = p.communicate() + # TODO: Use p.communicate(timeout=3600) if Python3 only + if p.returncode != 0: + # Show failing code, output and errors before returning + print("ERROR") + print("-- stdout " + "-" * (79 - 10)) + print(str(stdout, "utf-8")) + print("-- stderr " + "-" * (79 - 10)) + print(str(stderr, "utf-8")) + print("-" * 79) + return False + except KeyboardInterrupt: + p.terminate() + print("ABORTED") + sys.exit(1) + + # Sucessfully run + print("ok") + return True + + +def export_notebook(ipath, opath): + """ + Exports the notebook at `ipath` to a python file at `opath`. + """ + import nbconvert + from traitlets.config import Config + + # Create nbconvert configuration to ignore text cells + c = Config() + c.TemplateExporter.exclude_markdown = True + + # Load notebook, convert to python + e = nbconvert.exporters.PythonExporter(config=c) + code, __ = e.from_filename(ipath) + + # Remove "In [1]:" comments + r = re.compile(r"(\s*)# In\[([^]]*)\]:(\s)*") + code = r.sub("\n\n", code) + + # Store as executable script file + with open(opath, "w") as f: + f.write("#!/usr/bin/env python") + f.write(code) + os.chmod(opath, 0o775) + + +def run_doc_tests(): + """ + Checks if the documentation can be built, runs any doctests (currently not + used). + """ + print("Checking if docs can be built.") + p = subprocess.Popen( + ["sphinx-build", "-b", "doctest", "docs", "docs/build/html", "-W"] + ) + try: + ret = p.wait() + except KeyboardInterrupt: + try: + p.terminate() + except OSError: + pass + p.wait() + print("") + sys.exit(1) + if ret != 0: + print("FAILED") + sys.exit(ret) + + +if __name__ == "__main__": + run_notebook_and_scripts() + run_doc_tests() + # export_notebook("docs/examples/example.ipynb", "test.py") diff --git a/setup.cfg b/setup.cfg index 5868bbc..00afe46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,9 +16,41 @@ addopts = [flake8] max-line-length = 88 -exclude = .venv/,.eggs/ -extend-ignore = - E203, +exclude= + .git, + venv, +ignore= + # False positive for white space before ':' on list slice + # black should format these correctly + E203, + + # Block comment should start with '# ' + # Not if it's a commented out line + E265, + + # Ambiguous variable names + # It's absolutely fine to have i and I + E741, + + # List comprehension redefines variable + # Re-using throw-away variables like `i`, `x`, etc. is a Good Idea + F812, + + # Blank line at end of file + # This increases readability + W391, + + # Line break before binary operator + # This is now actually advised in pep8 + W503, + + # Line break after binary operator + W504, + + # Invalid escape sequence + # These happen all the time in latex parts of docstrings, + # e.g. \sigma + W605, [coverage:run] omit = r2t2/__init__.py