WIP: More transformers

.gitignore

@@ -63,3 +63,9 @@ target/
 .*.swp
 example
 tests/functional/output*py
+
+# mypy
+.mypy_cache/
+
+# Misc
+/test*.py

LICENSE.md

@@ -0,0 +1,23 @@
+
+# The MIT License (MIT)
+
+Copyright © 2017 by [nvbn](https://github.com/nvbn/).
+Copyright © 2019 by luk3yx.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,40 +1,73 @@
 # Py-backwards [![Build Status](https://travis-ci.org/nvbn/py-backwards.svg?branch=master)](https://travis-ci.org/nvbn/py-backwards)
 
-Python to python compiler that allows you to use some Python 3.6 features in older versions, you can try it in [the online demo](https://py-backwards.herokuapp.com/).
+Python to python compiler that allows you to use some Python 3.6+ features in older versions, you can try it in [the online demo](https://py-backwards.herokuapp.com/).
 
-Requires Python 3.3+ to run, can compile down to 2.7.
+Requires Python 3.3+ to run, can compile down to 2.7 (and down to 2.5 if you
+only use a subset of Python 3).
+
+Note that py_backwards creates variables beginning with `_py_backwards` for
+internal use, to prevent variable conflicts try to avoid function/variable
+names beginning with `_py_backwards` in your code.
 
 ## Supported features
 
+Target 3.7:
+* [Certain walrus operators](https://docs.python.org/3.8/whatsnew/3.8.html#assignment-expressions) - This is rather hit and miss, and some walrus operators currently
+    only work on CPython and only if the variable has already been defined in
+    the same scope.
+* [Positional only parameters](https://docs.python.org/3.8/whatsnew/3.8.html#positional-only-parameters)
+* [Self-documenting f-string expressions](https://docs.python.org/3.8/whatsnew/3.8.html#f-strings-support-for-self-documenting-expressions-and-debugging) (works automatically)
+
 Target 3.5:
-* [formatted string literals](https://docs.python.org/3/whatsnew/3.6.html#pep-498-formatted-string-literals) like `f'hi {x}'`
-* [variables annotations](https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep526) like `x: int = 10` and `x: int`
-* [underscores in numeric literals](https://docs.python.org/3/whatsnew/3.6.html#pep-515-underscores-in-numeric-literals) like `1_000_000` (works automatically)
+* [Formatted string literals](https://docs.python.org/3/whatsnew/3.6.html#pep-498-formatted-string-literals) like `f'hi {x}'`
+* [Variable annotations](https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep526) like `x: int = 10` and `x: int`
+* [Asynchronous generators](https://www.python.org/dev/peps/pep-0525)
+* [Underscores in numeric literals](https://docs.python.org/3/whatsnew/3.6.html#pep-515-underscores-in-numeric-literals) like `1_000_000` (works automatically)
 
 Target 3.4:
-* [starred unpacking](https://docs.python.org/3/whatsnew/3.5.html#pep-448-additional-unpacking-generalizations) like `[*range(1, 5), *range(10, 15)]` and `print(*[1, 2], 3, *[4, 5])`
-* [dict unpacking](https://docs.python.org/3/whatsnew/3.5.html#pep-448-additional-unpacking-generalizations) like `{1: 2, **{3: 4}}`
+* [Starred unpacking](https://docs.python.org/3/whatsnew/3.5.html#pep-448-additional-unpacking-generalizations) like `[*range(1, 5), *range(10, 15)]` and `print(*[1, 2], 3, *[4, 5])`
+* [Dict unpacking](https://docs.python.org/3/whatsnew/3.5.html#pep-448-additional-unpacking-generalizations) like `{1: 2, **{3: 4}}`
 
 Target 3.3:
-* import [pathlib2](https://pypi.python.org/pypi/pathlib2/) instead of pathlib
+* Import [pathlib2](https://pypi.python.org/pypi/pathlib2/) instead of pathlib
 
 Target 3.2:
-* [yield from](https://docs.python.org/3/whatsnew/3.3.html#pep-380)
-* [return from generator](https://docs.python.org/3/whatsnew/3.3.html#pep-380)
+* [`yield from`](https://docs.python.org/3/whatsnew/3.3.html#pep-380)
+* [Return from generator](https://docs.python.org/3/whatsnew/3.3.html#pep-380)
 
 Target 2.7:
-* [functions annotations](https://www.python.org/dev/peps/pep-3107/) like `def fn(a: int) -> str`
-* [imports from `__future__`](https://docs.python.org/3/howto/pyporting.html#prevent-compatibility-regressions)
-* [super without arguments](https://www.python.org/dev/peps/pep-3135/)
-* classes without base like `class A: pass`
-* imports from [six moves](https://pythonhosted.org/six/#module-six.moves)
-* metaclass
-* string/unicode literals (works automatically)
+* [Keyword only arguments](https://www.python.org/dev/peps/pep-3102/)
+* [Function annotations](https://www.python.org/dev/peps/pep-3107/) like `def fn(a: int) -> str`
+* [Imports from `__future__`](https://docs.python.org/3/howto/pyporting.html#prevent-compatibility-regressions)
+* [`super()` without arguments](https://www.python.org/dev/peps/pep-3135/)
+* [The `nonlocal` statement](https://www.python.org/dev/peps/pep-3104/),
+    provided you don't try and check for variables used with `nonlocal` in
+    `locals()`.
+* Implicit `object` class base.
+* Imports from [six.moves](https://pythonhosted.org/six/#module-six.moves)
+* Metaclasses
+* A `__nonzero__` alias for any `__bool__` methods.
+* String/unicode literals (works automatically)
 * `str` to `unicode`
-* define encoding (not transformer)
+* Add `# -*- coding: utf-8 -*-` (not transformer)
 * `dbm => anydbm` and `dbm.ndbm => dbm`
+* [Non-ASCII identifiers](https://www.python.org/dev/peps/pep-3131/). Non-ASCII
+    identifiers are mangled currently mangled in a similar way to
+    [Hy](https://docs.hylang.org/en/stable/language/syntax.html#mangling).
+
+Target 2.6:
+* Class decorators
+* Dict comprehension
+* Set literals
+
+Target 2.5:
+* `six.print_()` instead of `print()`.
+* `six.advance_iterator()` instead of `next()`.
+* `except as` (note that this breaks compatibility with Python 3.0+).
+* Keyword arguments after `*args`.
+* An `itertools.zip_longest` backport.
 
-For example, if you have some python 3.6 code, like:
+For example, if you have some Python 3.6 code, like:
 
 ```python
 def returning_range(x: int):
@@ -80,7 +113,7 @@ print(ImportantNumberManager().ten())
 print(ImportantNumberManager.eleven())
 ```
 
-You can compile it for python 2.7 with:
+You can compile it for Python 2.7 with:
 
 ```bash
 ➜ py-backwards -i input.py -o output.py -t 2.7
@@ -154,7 +187,7 @@ pip install py-backwards-packager
 ```
 
 And change `setup` import in `setup.py` to:
- 
+
 ```python
 try:
     from py_backwards_packager import setup
@@ -163,7 +196,7 @@ except ImportError:
 ```
 
 By default all targets enabled, but you can limit them with:
- 
+
 ```python
 setup(...,
       py_backwards_targets=['2.7', '3.3'])
@@ -263,7 +296,7 @@ from ..utils.snippet import snippet, let, extend
 def my_snippet(class_name, class_body):
     class class_name:  # will be replaced with `class_name`
         extend(class_body)  # body of the class will be extended with `class_body`
-        
+
         def fn(self):
             let(x)  # x will be replaced everywhere with unique name, like `_py_backwards_x_1`
             x = 10
@@ -286,5 +319,6 @@ it contains such useful functions like `find`, `get_parent` and etc.
 * [tox-py-backwards](https://github.com/nvbn/tox-py-backwards)
 * [py-backwards-packager](https://github.com/nvbn/py-backwards-packager)
 * [pytest-docker-pexpect](https://github.com/nvbn/pytest-docker-pexpect)
+* [lib3to6](https://gitlab.com/mbarkhau/lib3to6)
 
 ## License MIT

py_backwards/__init__.py

@@ -0,0 +1,42 @@
+
+import sys
+if sys.version_info >= (3, 9):
+    import ast
+    def unparse(tree):
+        return ast.unparse(ast.fix_missing_locations(tree))
+elif sys.version_info >= (3, 8):
+    import ast
+
+    # A hack to allow astunparse to parse ast.Constant-s.
+    import astunparse
+    from astunparse import unparse
+    from types import SimpleNamespace as _SimpleNamespace
+
+    def _Constant(self, tree):
+        value = tree.value
+        if isinstance(value, str):
+            self._Str(_SimpleNamespace(s=value))
+        elif isinstance(value, bytes):
+            self._Bytes(_SimpleNamespace(s=value))
+        elif value is Ellipsis:
+            self._Ellipsis(tree)
+        else:
+            self._NameConstant(tree)
+
+    def _NamedExpr(self, tree):
+        self.write('(')
+        self.dispatch(tree.target)
+        self.write(' := ')
+        self.dispatch(tree.value)
+        self.write(')')
+
+
+    if not hasattr(astunparse.Unparser, '_Constant'):
+        astunparse.Unparser._Constant = _Constant
+    if not hasattr(astunparse.Unparser, '_NamedExpr'):
+        astunparse.Unparser._NamedExpr = _NamedExpr
+
+    del _Constant, _NamedExpr
+else:
+    from typed_ast import ast3 as ast
+    from astunparse import unparse

py_backwards/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from .main import main
+
+if __name__ == '__main__':
+    main()

py_backwards/compiler.py

@@ -2,15 +2,17 @@
 from time import time
 from traceback import format_exc
 from typing import List, Tuple, Optional
-from typed_ast import ast3 as ast
-from astunparse import unparse, dump
+try:
+    from ast import dump
+except ImportError:
+    from astunparse import dump
 from autopep8 import fix_code
 from .files import get_input_output_paths, InputOutput
 from .transformers import transformers
 from .types import CompilationTarget, CompilationResult
 from .exceptions import CompilationError, TransformationError
 from .utils.helpers import debug
-from . import const
+from . import ast, const, unparse
 
 
 def _transform(path: str, code: str, target: CompilationTarget) -> Tuple[str, List[str]]:
@@ -30,6 +32,17 @@ def _transform(path: str, code: str, target: CompilationTarget) -> Tuple[str, Li
         working_tree = deepcopy(tree)
         try:
             result = transformer.transform(working_tree)
+        except SyntaxError as exc:
+            if isinstance(getattr(exc, 'ast_node', None), ast.AST):
+                if not getattr(exc.ast_node, 'lineno', None): # type: ignore
+                    ast.fix_missing_locations(working_tree)
+                exc.lineno = getattr(exc.ast_node, 'lineno', 0) # type: ignore
+                exc.offset = getattr(exc.ast_node, 'col_offset', -1) + 1 # type: ignore
+            else:
+                exc.lineno = exc.lineno or 0
+                exc.offset = exc.offset or 0
+
+            raise exc
         except:
             raise TransformationError(path, transformer,
                                       dump(tree), format_exc())
@@ -49,7 +62,10 @@ def _transform(path: str, code: str, target: CompilationTarget) -> Tuple[str, Li
             raise TransformationError(path, transformer,
                                       dump(tree), format_exc())
 
-    return fix_code(code), dependencies
+    # Disable E402 (moving imports to the top of the file) as it breaks.
+    code = fix_code(code, options={'ignore': ['E226', 'E24', 'W50', 'W690',
+                                              'E402']})
+    return code, dependencies
 
 
 def _compile_file(paths: InputOutput, target: CompilationTarget) -> List[str]:
@@ -62,14 +78,14 @@ def _compile_file(paths: InputOutput, target: CompilationTarget) -> List[str]:
                                                code, target)
     except SyntaxError as e:
         raise CompilationError(paths.input.as_posix(),
-                               code, e.lineno, e.offset)
+                               code, e.lineno, e.offset or 0)
 
     try:
         paths.output.parent.mkdir(parents=True)
     except FileExistsError:
         pass
 
-    if target == const.TARGETS['2.7']:
+    if target <= const.TARGETS['2.7']:
         transformed = '# -*- coding: utf-8 -*-\n{}'.format(transformed)
 
     with paths.output.open('w') as f:

py_backwards/const.py

@@ -1,14 +1,18 @@
 from collections import OrderedDict
 
-TARGETS = OrderedDict([('2.7', (2, 7)),
+TARGETS = OrderedDict([('2.5', (2, 5)),
+                       ('2.6', (2, 6)),
+                       ('2.7', (2, 7)),
                        ('3.0', (3, 0)),
                        ('3.1', (3, 1)),
                        ('3.2', (3, 2)),
                        ('3.3', (3, 3)),
                        ('3.4', (3, 4)),
                        ('3.5', (3, 5)),
-                       ('3.6', (3, 6))])
+                       ('3.6', (3, 6)),
+                       ('3.7', (3, 7)),
+                       ('3.8', (3, 8))])
 
 SYNTAX_ERROR_OFFSET = 5
 
-TARGET_ALL = (9999, 9999)
+TARGET_ALL = next(reversed(TARGETS.values()))

py_backwards/main.py

@@ -3,20 +3,34 @@
 init()
 
 from argparse import ArgumentParser
+import atexit
+import pathlib
+import shutil
 import sys
+import tempfile
 from .compiler import compile_files
 from .conf import init_settings
 from . import const, messages, exceptions
 
+def _cleanup(tmpdir):
+    try:
+        print('\n# -*- coding: utf-8 -*-')
+        for path in pathlib.Path(tmpdir).glob('**/*.py'):
+            print()
+            print('# ----------', path.name, '---------- #')
+            with path.open('r') as f:
+                shutil.copyfileobj(f, sys.stdout)
+            path.unlink()
+    finally:
+        shutil.rmtree(tmpdir)
 
 def main() -> int:
-    parser = ArgumentParser(
-        'py-backwards',
+    parser = ArgumentParser('py-backwards',
         description='Python to python compiler that allows you to use some '
-                    'Python 3.6 features in older versions.')
+                    'Python 3.6+ features in older versions.')
     parser.add_argument('-i', '--input', type=str, nargs='+', required=True,
                         help='input file or folder')
-    parser.add_argument('-o', '--output', type=str, required=True,
+    parser.add_argument('-o', '--output', type=str, default='-',
                         help='output file or folder')
     parser.add_argument('-t', '--target', type=str,
                         required=True, choices=const.TARGETS.keys(),
@@ -28,9 +42,17 @@ def main() -> int:
     args = parser.parse_args()
     init_settings(args)
 
+    output = args.output
+    if output == '-':
+        output = tempfile.mkdtemp()
+        atexit.register(_cleanup, output)
+        result_file = sys.stderr
+    else:
+        result_file = sys.stdout
+
     try:
         for input_ in args.input:
-            result = compile_files(input_, args.output,
+            result = compile_files(input_, output,
                                    const.TARGETS[args.target],
                                    args.root)
     except exceptions.CompilationError as e:
@@ -50,5 +72,5 @@ def main() -> int:
         print(messages.permission_error(args.output), file=sys.stderr)
         return 1
 
-    print(messages.compilation_result(result))
+    print(messages.compilation_result(result), file=result_file)
     return 0

py_backwards/messages.py

@@ -53,7 +53,7 @@ def syntax_error(e: CompilationError) -> str:
         red=Fore.RED,
         e=e,
         reset=Style.RESET_ALL,
-        bright=Style.BRIGHT,
+        # bright=Style.BRIGHT,
         lines='\n'.join(lines))