update datalad buildsupport scrypts

_datalad_buildsupport/formatters.py

@@ -7,7 +7,10 @@
 
 import argparse
 import datetime
+import os
 import re
+import time
+from textwrap import wrap
 
 
 class ManPageFormatter(argparse.HelpFormatter):
@@ -24,7 +27,7 @@ def __init__(self,
                  authors=None,
                  version=None
                  ):
-
+        from datalad import cfg
         super(ManPageFormatter, self).__init__(
             prog,
             indent_increment=indent_increment,
@@ -33,7 +36,10 @@ def __init__(self,
 
         self._prog = prog
         self._section = 1
-        self._today = datetime.date.today().strftime('%Y\\-%m\\-%d')
+        self._today = datetime.datetime.fromtimestamp(
+            cfg.obtain('datalad.source.epoch'),
+            datetime.timezone.utc
+        ).strftime('%Y\\-%m\\-%d')
         self._ext_sections = ext_sections
         self._version = version
 
@@ -75,7 +81,7 @@ def _mk_title(self, prog):
 
     def _mk_name(self, prog, desc):
         """
-        this method is in consitent with others ... it relies on
+        this method is in consistent with others ... it relies on
         distribution
         """
         desc = desc.splitlines()[0] if desc else 'it is in the name'
@@ -195,7 +201,9 @@ def _mk_synopsis(self, parser):
                                    parser._mutually_exclusive_groups, '')
 
         usage = usage.replace('%s ' % self._prog, '')
-        usage = 'Synopsis\n--------\n::\n\n  %s %s\n' \
+        usage = '\n'.join(wrap(
+            usage, break_on_hyphens=False, subsequent_indent=6*' '))
+        usage = 'Synopsis\n--------\n::\n\n  %s %s\n\n' \
                 % (self._markup(self._prog), usage)
         return usage
 
@@ -251,7 +259,7 @@ def _mk_options(self, parser):
 
     def _format_action(self, action):
         # determine the required width and the entry label
-        action_header = self._format_action_invocation(action)
+        action_header = self._format_action_invocation(action, doubledash='-\\-')
 
         if action.help:
             help_text = self._expand_help(action)

_datalad_buildsupport/setup.py

@@ -8,19 +8,51 @@
 
 import datetime
 import os
-
-from os.path import (
-    dirname,
-    join as opj,
+import platform
+import sys
+from os import (
+    linesep,
+    makedirs,
 )
-from setuptools import Command, DistutilsOptionError
-from setuptools.config import read_configuration
-
-import versioneer
+from os.path import dirname
+from os.path import join as opj
+from os.path import sep as pathsep
+from os.path import splitext
+
+import setuptools
+from genericpath import exists
+from packaging.version import Version
+from setuptools import (
+    Command,
+    find_namespace_packages,
+    findall,
+    setup,
+)
+from setuptools.errors import OptionError
 
 from . import formatters as fmt
 
 
+def _path_rel2file(*p):
+    # dirname instead of joining with pardir so it works if
+    # datalad_build_support/ is just symlinked into some extension
+    # while developing
+    return opj(dirname(dirname(__file__)), *p)
+
+
+def get_version(name):
+    """Determine version via importlib_metadata
+
+    Parameters
+    ----------
+    name: str
+      Name of the folder (package) where from to read version.py
+    """
+    # delay import so we do not require it for a simple setup stage
+    from importlib.metadata import version as importlib_version
+    return importlib_version(name)
+
+
 class BuildManPage(Command):
     # The BuildManPage code was originally distributed
     # under the same License of Python
@@ -29,33 +61,27 @@ class BuildManPage(Command):
     description = 'Generate man page from an ArgumentParser instance.'
 
     user_options = [
-        ('manpath=', None,
-         'output path for manpages (relative paths are relative to the '
-         'datalad package)'),
-        ('rstpath=', None,
-         'output path for RST files (relative paths are relative to the '
-         'datalad package)'),
+        ('manpath=', None, 'output path for manpages'),
+        ('rstpath=', None, 'output path for RST files'),
         ('parser=', None, 'module path to an ArgumentParser instance'
          '(e.g. mymod:func, where func is a method or function which return'
          'a dict with one or more arparse.ArgumentParser instances.'),
-        ('cmdsuite=', None, 'module path to an extension command suite '
-         '(e.g. mymod:command_suite) to limit the build to the contained '
-         'commands.'),
     ]
 
     def initialize_options(self):
         self.manpath = opj('build', 'man')
         self.rstpath = opj('docs', 'source', 'generated', 'man')
-        self.parser = 'datalad.cmdline.main:setup_parser'
-        self.cmdsuite = None
+        self.parser = 'datalad.cli.parser:setup_parser'
 
     def finalize_options(self):
         if self.manpath is None:
-            raise DistutilsOptionError('\'manpath\' option is required')
+            raise OptionError('\'manpath\' option is required')
         if self.rstpath is None:
-            raise DistutilsOptionError('\'rstpath\' option is required')
+            raise OptionError('\'rstpath\' option is required')
         if self.parser is None:
-            raise DistutilsOptionError('\'parser\' option is required')
+            raise OptionError('\'parser\' option is required')
+        self.manpath = _path_rel2file(self.manpath)
+        self.rstpath = _path_rel2file(self.rstpath)
         mod_name, func_name = self.parser.split(':')
         fromlist = mod_name.split('.')
         try:
@@ -64,18 +90,10 @@ def finalize_options(self):
                 ['datalad'],
                 formatter_class=fmt.ManPageFormatter,
                 return_subparsers=True,
-                # ignore extensions only for the main package to avoid pollution
-                # with all extension commands that happen to be installed
-                help_ignore_extensions=self.distribution.get_name() == 'datalad')
+                help_ignore_extensions=True)
 
         except ImportError as err:
             raise err
-        if self.cmdsuite:
-            mod_name, suite_name = self.cmdsuite.split(':')
-            mod = __import__(mod_name, fromlist=mod_name.split('.'))
-            suite = getattr(mod, suite_name)
-            self.cmdlist = [c[2] if len(c) > 2 else c[1].replace('_', '-').lower()
-                            for c in suite[1]]
 
         self.announce('Writing man page(s) to %s' % self.manpath)
         self._today = datetime.date.today()
@@ -125,29 +143,24 @@ def run(self):
         #appname = self._parser.prog
         appname = 'datalad'
 
-        cfg = read_configuration(
-            opj(dirname(dirname(__file__)), 'setup.cfg'))['metadata']
-
         sections = {
             'Authors': """{0} is developed by {1} <{2}>.""".format(
-                appname, cfg['author'], cfg['author_email']),
+                appname, dist.get_author(), dist.get_author_email()),
         }
 
         for cls, opath, ext in ((fmt.ManPageFormatter, self.manpath, '1'),
                                 (fmt.RSTManPageFormatter, self.rstpath, 'rst')):
             if not os.path.exists(opath):
                 os.makedirs(opath)
             for cmdname in getattr(self, 'cmdline_names', list(self._parser)):
-                if hasattr(self, 'cmdlist') and cmdname not in self.cmdlist:
-                    continue
                 p = self._parser[cmdname]
                 cmdname = "{0}{1}".format(
                     'datalad ' if cmdname != 'datalad' else '',
                     cmdname)
                 format = cls(
                     cmdname,
                     ext_sections=sections,
-                    version=versioneer.get_version())
+                    version=get_version(getattr(self, 'mod_name', appname)))
                 formatted = format.format_man_page(p)
                 with open(opj(opath, '{0}.{1}'.format(
                         cmdname.replace(' ', '-'),
@@ -156,6 +169,42 @@ def run(self):
                     f.write(formatted)
 
 
+class BuildRSTExamplesFromScripts(Command):
+    description = 'Generate RST variants of example shell scripts.'
+
+    user_options = [
+        ('expath=', None, 'path to look for example scripts'),
+        ('rstpath=', None, 'output path for RST files'),
+    ]
+
+    def initialize_options(self):
+        self.expath = opj('docs', 'examples')
+        self.rstpath = opj('docs', 'source', 'generated', 'examples')
+
+    def finalize_options(self):
+        if self.expath is None:
+            raise OptionError('\'expath\' option is required')
+        if self.rstpath is None:
+            raise OptionError('\'rstpath\' option is required')
+        self.expath = _path_rel2file(self.expath)
+        self.rstpath = _path_rel2file(self.rstpath)
+        self.announce('Converting example scripts')
+
+    def run(self):
+        opath = self.rstpath
+        if not os.path.exists(opath):
+            os.makedirs(opath)
+
+        from glob import glob
+        for example in glob(opj(self.expath, '*.sh')):
+            exname = os.path.basename(example)[:-3]
+            with open(opj(opath, '{0}.rst'.format(exname)), 'w') as out:
+                fmt.cmdline_example_to_rst(
+                    open(example),
+                    out=out,
+                    ref='_example_{0}'.format(exname))
+
+
 class BuildConfigInfo(Command):
     description = 'Generate RST documentation for all config items.'
 
@@ -168,16 +217,17 @@ def initialize_options(self):
 
     def finalize_options(self):
         if self.rstpath is None:
-            raise DistutilsOptionError('\'rstpath\' option is required')
+            raise OptionError('\'rstpath\' option is required')
+        self.rstpath = _path_rel2file(self.rstpath)
         self.announce('Generating configuration documentation')
 
     def run(self):
         opath = self.rstpath
         if not os.path.exists(opath):
             os.makedirs(opath)
 
-        from datalad.interface.common_cfg import definitions as cfgdefs
         from datalad.dochelpers import _indent
+        from datalad.interface.common_cfg import definitions as cfgdefs
 
         categories = {
             'global': {},
@@ -218,3 +268,100 @@ def run(self):
                         desc_tmpl += 'undocumented\n'
                     v.update(docs)
                     rst.write(_indent(desc_tmpl.format(**v), '    '))
+
+
+def get_long_description_from_README():
+    """Read README.md, convert to .rst using pypandoc
+
+    If pypandoc is not available or fails - just output original .md.
+
+    Returns
+    -------
+    dict
+      with keys long_description and possibly long_description_content_type
+      for newer setuptools which support uploading of markdown as is.
+    """
+    # PyPI used to not render markdown. Workaround for a sane appearance
+    # https://github.com/pypa/pypi-legacy/issues/148#issuecomment-227757822
+    # is still in place for older setuptools
+
+    README = opj(_path_rel2file('README.md'))
+
+    ret = {}
+    if Version(setuptools.__version__) >= Version('38.6.0'):
+        # check than this
+        ret['long_description'] = open(README).read()
+        ret['long_description_content_type'] = 'text/markdown'
+        return ret
+
+    # Convert or fall-back
+    try:
+        import pypandoc
+        return {'long_description': pypandoc.convert(README, 'rst')}
+    except (ImportError, OSError) as exc:
+        # attempting to install pandoc via brew on OSX currently hangs and
+        # pypandoc imports but throws OSError demanding pandoc
+        print(
+                "WARNING: pypandoc failed to import or thrown an error while "
+                "converting"
+                " README.md to RST: %r   .md version will be used as is" % exc
+        )
+        return {'long_description': open(README).read()}
+
+
+def findsome(subdir, extensions):
+    """Find files under subdir having specified extensions
+
+    Leading directory (datalad) gets stripped
+    """
+    return [
+        f.split(pathsep, 1)[1] for f in findall(opj('datalad', subdir))
+        if splitext(f)[-1].lstrip('.') in extensions
+    ]
+
+
+def datalad_setup(name, **kwargs):
+    """A helper for a typical invocation of setuptools.setup.
+
+    If not provided in kwargs, following fields will be autoset to the defaults
+    or obtained from the present on the file system files:
+
+    - author
+    - author_email
+    - packages -- all found packages which start with `name`
+    - long_description -- converted to .rst using pypandoc README.md
+    - version -- parsed `__version__` within `name/version.py`
+
+    Parameters
+    ----------
+    name: str
+        Name of the Python package
+    **kwargs:
+        The rest of the keyword arguments passed to setuptools.setup as is
+    """
+    # Simple defaults
+    for k, v in {
+        'author': "The DataLad Team and Contributors",
+        'author_email': "[email protected]"
+    }.items():
+        if kwargs.get(k) is None:
+            kwargs[k] = v
+
+    # More complex, requiring some function call
+
+    # Only recentish versions of find_packages support include
+    # packages = find_packages('.', include=['datalad*'])
+    # so we will filter manually for maximal compatibility
+    if kwargs.get('packages') is None:
+        # Use find_namespace_packages() in order to include folders that
+        # contain data files but no Python code
+        kwargs['packages'] = [pkg for pkg in find_namespace_packages('.') if pkg.startswith(name)]
+    if kwargs.get('long_description') is None:
+        kwargs.update(get_long_description_from_README())
+
+    cmdclass = kwargs.get('cmdclass', {})
+    # Check if command needs some module specific handling
+    for v in cmdclass.values():
+        if hasattr(v, 'handle_module'):
+            getattr(v, 'handle_module')(name, **kwargs)
+    return setup(name=name, **kwargs)