[FEATURE] Decorators for API docs (part 2) (great-expectations#6497)

Author: anthonyburdi

docs/sphinx_api_docs_source/requirements-dev-api-docs.txt

@@ -1,3 +1,4 @@
+docstring-parser==0.15
 myst-parser
 pydata-sphinx-theme==0.11.0  # Pinned to keep the css styling consistent.
 sphinx~=4.5.0

docs/sphinx_api_docs_source/tasks.py

@@ -39,7 +39,7 @@ def _exit_with_error_if_not_run_from_correct_dir(
             os.path.realpath(os.path.dirname(os.path.realpath(__file__)))
         )
     curdir = pathlib.Path(os.path.realpath(os.getcwd()))
-    exit_message = f"The {task_name} task must be invoked from the same directory as the task.py file at the top of the repo."
+    exit_message = f"The {task_name} task must be invoked from the same directory as the tasks.py file."
     if correct_dir != curdir:
         raise invoke.Exit(
             exit_message,

great_expectations/core/_docs_decorators.py

@@ -1,11 +1,25 @@
-from typing import Callable
+from textwrap import dedent
+from typing import Any, Callable, TypeVar
+
+try:
+    import docstring_parser
+except ImportError:
+    docstring_parser = None
 
 WHITELISTED_TAG = "--Public API--"
 
+F = TypeVar("F", bound=Callable[..., Any])
+
 
 def public_api(func) -> Callable:
     """Add the public API tag for processing by the auto documentation generator.
 
+    Used as a decorator:
+
+        @public_api
+        def my_method(some_argument):
+            ...
+
     This tag is added at import time.
     """
 
@@ -14,3 +28,234 @@ def public_api(func) -> Callable:
     func.__doc__ = WHITELISTED_TAG + existing_docstring
 
     return func
+
+
+def deprecated_method(
+    version: str,
+    message: str = "",
+):
+    """Add a deprecation warning to the docstring of the decorated method.
+
+    Used as a decorator:
+
+        @deprecated_method(version="1.2.3", message="Optional message")
+        def my_method(some_argument):
+            ...
+
+    Args:
+        version: Version number when the method was deprecated.
+        message: Optional deprecation message.
+    """
+
+    text = f".. deprecated:: {version}" "\n" f"    {message}"
+
+    def wrapper(func: F) -> F:
+        """Wrapper method that accepts func, so we can modify the docstring."""
+        return _add_text_to_function_docstring_after_summary(
+            func=func,
+            text=text,
+        )
+
+    return wrapper
+
+
+def new_method(
+    version: str,
+    message: str = "",
+):
+    """Add a version added note to the docstring of the decorated method.
+
+    Used as a decorator:
+
+        @new_method(version="1.2.3", message="Optional message")
+        def my_method(some_argument):
+            ...
+
+    Args:
+        version: Version number when the method was added.
+        message: Optional message.
+    """
+
+    text = f".. versionadded:: {version}" "\n" f"    {message}"
+
+    def wrapper(func: F) -> F:
+        """Wrapper method that accepts func, so we can modify the docstring."""
+        return _add_text_to_function_docstring_after_summary(
+            func=func,
+            text=text,
+        )
+
+    return wrapper
+
+
+def deprecated_argument(
+    argument_name: str,
+    version: str,
+    message: str = "",
+):
+    """Add an arg-specific deprecation warning to the docstring of the decorated method.
+
+    Used as a decorator:
+
+        @deprecated_argument(argument_name="some_argument", version="1.2.3", message="Optional message")
+        def my_method(some_argument):
+            ...
+
+    If docstring_parser is not installed, this will not modify the docstring.
+
+    Args:
+        argument_name: Name of the argument to associate with the deprecation note.
+        version: Version number when the method was deprecated.
+        message: Optional deprecation message.
+    """
+
+    text = f".. deprecated:: {version}" "\n" f"    {message}"
+
+    def wrapper(func: F) -> F:
+        """Wrapper method that accepts func, so we can modify the docstring."""
+        if not docstring_parser:
+            return func
+
+        return _add_text_below_function_docstring_argument(
+            func=func,
+            argument_name=argument_name,
+            text=text,
+        )
+
+    return wrapper
+
+
+def new_argument(
+    argument_name: str,
+    version: str,
+    message: str = "",
+):
+    """Add note for new arguments about which version the argument was added.
+
+    Used as a decorator:
+
+        @new_argument(argument_name="some_argument", version="1.2.3", message="Optional message")
+        def my_method(some_argument):
+            ...
+
+    If docstring_parser is not installed, this will not modify the docstring.
+
+    Args:
+        argument_name: Name of the argument to associate with the note.
+        version: The version number to associate with the note.
+        message: Optional message.
+    """
+
+    text = f".. versionadded:: {version}" "\n" f"    {message}"
+
+    def wrapper(func: F) -> F:
+        """Wrapper method that accepts func, so we can modify the docstring."""
+        if not docstring_parser:
+            return func
+
+        return _add_text_below_function_docstring_argument(
+            func=func,
+            argument_name=argument_name,
+            text=text,
+        )
+
+    return wrapper
+
+
+def _add_text_to_function_docstring_after_summary(func: F, text: str) -> F:
+    """Insert text into docstring, e.g. rst directive.
+
+    Args:
+        func: Add text to provided func docstring.
+        text: String to add to the docstring, can be a rst directive e.g.:
+            text = (
+                ".. versionadded:: 1.2.3\n"
+                "    Added in version 1.2.3\n"
+            )
+
+    Returns:
+        func with modified docstring.
+    """
+    existing_docstring = func.__doc__ if func.__doc__ else ""
+    split_docstring = existing_docstring.split("\n", 1)
+
+    docstring = ""
+    if len(split_docstring) == 2:
+        short_description, docstring = split_docstring
+        docstring = (
+            f"{short_description.strip()}\n"
+            "\n"
+            f"{text}\n"
+            "\n"
+            f"{dedent(docstring)}"
+        )
+    elif len(split_docstring) == 1:
+        short_description = split_docstring[0]
+        docstring = f"{short_description.strip()}\n" "\n" f"{text}\n"
+    elif len(split_docstring) == 0:
+        docstring = f"{text}\n"
+
+    func.__doc__ = docstring
+
+    return func
+
+
+def _add_text_below_function_docstring_argument(
+    func: F,
+    argument_name: str,
+    text: str,
+) -> F:
+    """Add text below specified docstring argument.
+
+    Args:
+        func: Function whose docstring will be modified.
+        argument_name: Name of the argument to add text to its description.
+        text: Text to add to the argument description.
+
+    Returns:
+        func with modified docstring.
+    """
+    existing_docstring = func.__doc__ if func.__doc__ else ""
+
+    func.__doc__ = _add_text_below_string_docstring_argument(
+        docstring=existing_docstring, argument_name=argument_name, text=text
+    )
+
+    return func
+
+
+def _add_text_below_string_docstring_argument(
+    docstring: str, argument_name: str, text: str
+) -> str:
+    """Add text below an argument in a docstring.
+
+    Note: Can be used for rst directives.
+
+    Args:
+        docstring: Docstring to modify.
+        argument_name: Argument to place text below.
+        text: Text to place below argument. Can be an rst directive.
+
+    Returns:
+        Modified docstring.
+    """
+    parsed_docstring = docstring_parser.parse(docstring)
+
+    if argument_name not in (param.arg_name for param in parsed_docstring.params):
+        raise ValueError(
+            f"Please specify an existing argument, you specified {argument_name}."
+        )
+
+    for param in parsed_docstring.params:
+        if param.arg_name == argument_name:
+            if param.description is None:
+                param.description = text
+            else:
+                param.description += "\n\n" + text + "\n\n"
+
+    # RenderingStyle.EXPANDED used to make sure any line breaks before and
+    # after the added text are included (for Sphinx html rendering).
+    return docstring_parser.compose(
+        docstring=parsed_docstring,
+        rendering_style=docstring_parser.RenderingStyle.EXPANDED,
+    )

great_expectations/core/usage_statistics/package_dependencies.py

@@ -71,6 +71,7 @@ class GXDependencies:
             "azure-storage-blob",
             "black",
             "boto3",
+            "docstring-parser",
             "feather-format",
             "flake8",
             "flask",
@@ -204,6 +205,8 @@ class GXDependencies:
         "uszipcode",
         "yahoo_fin",
         "zipcodes",
+        # requirements-dev-api-docs-test.txt
+        "docstring-parser",
     ]
 
     GX_DEV_DEPENDENCIES: Set[str] = set(ALL_GX_DEV_DEPENDENCIES) - set(

reqs/requirements-dev-api-docs-test.txt

@@ -0,0 +1 @@
+docstring-parser==0.15

reqs/requirements-dev-test.txt

@@ -1,2 +1,3 @@
 --requirement requirements-dev-lite.txt
 --requirement requirements-dev-contrib.txt
+--requirement requirements-dev-api-docs-test.txt

requirements-dev.txt

@@ -1,6 +1,5 @@
 --requirement requirements.txt
---requirement reqs/requirements-dev-lite.txt
---requirement reqs/requirements-dev-contrib.txt
+--requirement reqs/requirements-dev-test.txt
 --requirement reqs/requirements-dev-sqlalchemy.txt
 
 --requirement reqs/requirements-dev-arrow.txt

tasks.py

@@ -360,7 +360,7 @@ def _exit_with_error_if_not_in_repo_root(task_name: str):
     """Exit if the command was not run from the repository root."""
     filedir = os.path.realpath(os.path.dirname(os.path.realpath(__file__)))
     curdir = os.path.realpath(os.getcwd())
-    exit_message = f"The {task_name} task must be invoked from the same directory as the task.py file at the top of the repo."
+    exit_message = f"The {task_name} task must be invoked from the same directory as the tasks.py file at the top of the repo."
     if filedir != curdir:
         raise invoke.Exit(
             exit_message,