Generate docs for generic easyblocks (REVIEW)

easybuild/tools/docs.py

@@ -34,25 +34,29 @@
 @author: Ward Poelmans (Ghent University)
 """
 import copy
+import inspect
+import os
 
 from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, HIDDEN, sorted_categories
 from easybuild.framework.easyconfig.easyconfig import get_easyblock_class
 from easybuild.tools.ordereddict import OrderedDict
-from easybuild.tools.utilities import quote_str
+from easybuild.tools.utilities import quote_str, import_available_modules
+from easybuild.tools.filetools import read_file
 
 
 FORMAT_RST = 'rst'
 FORMAT_TXT = 'txt'
 
 
+def det_col_width(entries, title):
+    """Determine column width based on column title and list of entries."""
+    return max(map(len, entries + [title]))
+
+
 def avail_easyconfig_params_rst(title, grouped_params):
     """
     Compose overview of available easyconfig parameters, in RST format.
     """
-    def det_col_width(entries, title):
-        """Determine column width based on column title and list of entries."""
-        return max(map(len, entries + [title]))
-
     # main title
     lines = [
         title,
@@ -65,29 +69,14 @@ def det_col_width(entries, title):
         lines.append("%s parameters" % grpname)
         lines.extend(['-' * len(lines[-1]), ''])
 
-        name_title = "**Parameter name**"
-        descr_title = "**Description**"
-        dflt_title = "**Default value**"
-
-        # figure out column widths
-        nw = det_col_width(grouped_params[grpname].keys(), name_title) + 4  # +4 for raw format ("``foo``")
-        dw = det_col_width([x[0] for x in grouped_params[grpname].values()], descr_title)
-        dfw = det_col_width([str(quote_str(x[1])) for x in grouped_params[grpname].values()], dflt_title)
+        titles = ["**Parameter name**", "**Description**", "**Default value**"]
+        values = [
+            ['``' + name + '``' for name in grouped_params[grpname].keys()],  # parameter name
+            [x[0] for x in grouped_params[grpname].values()],  # description
+            [str(quote_str(x[1])) for x in grouped_params[grpname].values()]  #default value
+        ]
 
-        # 3 columns (name, description, default value), left-aligned, {c} is fill char
-        line_tmpl = "{0:{c}<%s}   {1:{c}<%s}   {2:{c}<%s}" % (nw, dw, dfw)
-        table_line = line_tmpl.format('', '', '', c='=', nw=nw, dw=dw, dfw=dfw)
-
-        # table header
-        lines.append(table_line)
-        lines.append(line_tmpl.format(name_title, descr_title, dflt_title, c=' '))
-        lines.append(line_tmpl.format('', '', '', c='-'))
-
-        # table rows by parameter
-        for name, (descr, dflt) in sorted(grouped_params[grpname].items()):
-            rawname = '``%s``' % name
-            lines.append(line_tmpl.format(rawname, descr, str(quote_str(dflt)), c=' '))
-        lines.append(table_line)
+        lines.extend(mk_rst_table(titles, values))
         lines.append('')
 
     return '\n'.join(lines)
@@ -160,3 +149,139 @@ def avail_easyconfig_params(easyblock, output_format):
         FORMAT_TXT: avail_easyconfig_params_txt,
     }
     return avail_easyconfig_params_functions[output_format](title, grouped_params)
+
+
+def gen_easyblocks_overview_rst(package_name, path_to_examples, common_params={}, doc_functions=[]):
+    """
+    Compose overview of all easyblocks in the given package in rst format
+    """
+    modules = import_available_modules(package_name)
+    docs = []
+    all_blocks = []
+
+    # get all blocks
+    for m in modules:
+        for name,obj in inspect.getmembers(m, inspect.isclass):
+            eb_class = getattr(m, name)
+            # skip imported classes that are not easyblocks
+            if eb_class.__module__.startswith(package_name) and eb_class not in all_blocks:
+                all_blocks.append(eb_class)
+
+    for eb_class in all_blocks:
+        docs.append(gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks))
+
+    title = 'Overview of generic easyblocks'
+
+    heading = ['=' * len(title), title, '=' * len(title), '', '.. contents::', '    :depth: 2', '']
+    return heading + sorted(docs)
+
+def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc_functions, all_blocks):
+    """
+    Compose overview of one easyblock given class object of the easyblock in rst format
+    """
+    classname = eb_class.__name__
+
+    lines = [
+        '.. _' + classname + ':',
+        '',
+        '``' + classname + '``',
+        '=' * (len(classname)+4),
+        '',
+    ]
+
+    bases = []
+    for b in eb_class.__bases__:
+        base = ':ref:`' + b.__name__ +'`' if b in all_blocks else b.__name__
+        bases.append(base)
+
+    derived = '(derives from ' + ', '.join(bases) + ')'
+    lines.extend([derived, ''])
+
+    # Description (docstring)
+    lines.extend([eb_class.__doc__.strip(), ''])
+
+    # Add extra options, if any
+    if eb_class.extra_options():
+        extra_parameters = 'Extra easyconfig parameters specific to ``' + classname + '`` easyblock'
+        lines.extend([extra_parameters, '-' * len(extra_parameters), ''])
+        ex_opt = eb_class.extra_options()
+
+        titles = ['easyconfig parameter', 'description', 'default value']
+        values = [
+            ['``' + key + '``' for key in ex_opt],  # parameter name
+            [val[1] for val in ex_opt.values()],  # description
+            ['``' + str(quote_str(val[0])) + '``' for val in ex_opt.values()]  # default value
+        ]
+
+        lines.extend(mk_rst_table(titles, values))
+
+    # Add commonly used parameters
+    if classname in common_params:
+        commonly_used = 'Commonly used easyconfig parameters with ``' + classname + '`` easyblock'
+        lines.extend([commonly_used, '-' * len(commonly_used)])
+
+        for opt in common_params[classname]:
+            param = '* ``' + opt + '`` - ' + DEFAULT_CONFIG[opt][1]
+            lines.append(param)
+    lines.append('')
+
+    # Add docstring for custom steps
+    custom = []
+    inh = ''
+    for func in doc_functions:
+        if func in eb_class.__dict__:
+            f = eb_class.__dict__[func]
+        elif func in eb_class.__bases__[0].__dict__:
+            f = eb_class.__bases__[0].__dict__[func]
+            inh = ' (inherited)'
+
+        if f.__doc__:
+            custom.append('* ``' + func + '`` - ' + f.__doc__.strip() + inh)
+
+    if custom:
+        title = 'Customised steps in ``' + classname + '`` easyblock'
+        lines.extend([title, '-' * len(title)] + custom)
+        lines.append('')
+
+    # Add example if available
+    if os.path.exists(os.path.join(path_to_examples, '%s.eb' % classname)):
+        title = 'Example for ``' + classname + '`` easyblock'
+        lines.extend(['', title, '-' * len(title), '', '::', ''])
+        for line in read_file(os.path.join(path_to_examples, classname+'.eb')).split('\n'):
+            lines.append('    ' + line.strip())
+        lines.append('') # empty line after literal block
+
+    return '\n'.join(lines)
+
+
+def mk_rst_table(titles, values):
+    """
+    Returns an rst table with given titles and values (a nested list of string values for each column)
+    """
+    num_col = len(titles)
+    table = []
+    col_widths = []
+    tmpl = []
+    line= []
+
+    # figure out column widths
+    for i in range(0, num_col):
+        col_widths.append(det_col_width(values[i], titles[i]))
+
+        # make line template
+        tmpl.append('{' + str(i) + ':{c}<' + str(col_widths[i]) + '}')
+        line.append('') # needed for table line
+
+    line_tmpl = '   '.join(tmpl)
+    table_line = line_tmpl.format(*line, c="=")
+
+    table.append(table_line)
+    table.append(line_tmpl.format(*titles, c=' '))
+    table.append(table_line)
+
+    for i in range(0, len(values[0])):
+        table.append(line_tmpl.format(*[v[i] for v in values], c=' '))
+
+    table.extend([table_line, ''])
+
+    return table

test/framework/docs.py

@@ -0,0 +1,65 @@
+# #
+# Copyright 2012-2015 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
+# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# http://github.com/hpcugent/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+Unit tests for docs.py.
+"""
+
+from test.framework.utilities import EnhancedTestCase, init_config
+from unittest import TestLoader, main
+from easybuild.tools.docs import mk_rst_table
+
+class DocsTest(EnhancedTestCase):
+
+    def test_rst_table(self):
+        """ Test mk_rst_table function """
+        entries_1 = [['one', 'two', 'three']]
+        t1 = 'This title is long'
+        titles_1 = [t1]
+
+        # small table
+        table_1 = '\n'.join(mk_rst_table(titles_1, entries_1))
+        check_1 = '\n'.join([
+            '=' * len(t1),
+            t1,
+            '=' * len(t1),
+            'one' + ' ' * (len(t1) - 3),
+            'two' + ' ' * (len(t1) -3),
+            'three' + ' ' * (len(t1) - 5),
+            '=' * len(t1),
+            '',
+        ])
+
+        self.assertEqual(table_1, check_1)
+
+
+def suite():
+    """ returns all test cases in this module """
+    return TestLoader().loadTestsFromTestCase(DocsTest)
+
+if __name__ == '__main__':
+    # also check the setUp for debug
+    # logToScreen(enable=True)
+    # setLogLevelDebug()
+    main()

test/framework/suite.py

@@ -62,6 +62,7 @@
 import test.framework.easyconfigformat as ef
 import test.framework.ebconfigobj as ebco
 import test.framework.easyconfigversion as ev
+import test.framework.docs as d
 import test.framework.filetools as f
 import test.framework.format_convert as f_c
 import test.framework.general as gen
@@ -103,7 +104,7 @@
 # call suite() for each module and then run them all
 # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
 tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw,
-         p, i, pkg]
+         p, i, pkg, d]
 
 SUITE = unittest.TestSuite([x.suite() for x in tests])