diff --git a/.travis.yml b/.travis.yml index 4e9f813..7f0a4b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,18 @@ # Config file for automatic testing at travis-ci.org - +sudo: false language: python python: - - "2.6" - "2.7" - - "3.2" - - "3.3" - "3.4" - "3.5" - "3.6" + - "3.7" + - "3.8" # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: pip install -r requirements.txt # command to run tests, e.g. python setup.py test script: python setup.py test +cache: pip diff --git a/AUTHORS.rst b/AUTHORS.rst index 9c21ecf..f0683ed 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -12,3 +12,9 @@ Authors * The Python Software Foundation * Bogdan Opanchuk +* Vladimir Iakovlev +* Thomas Grainger +* Amund Hov +* Jakub Wilk +* Mateusz Bysiek +* Serge Sans Paille diff --git a/HISTORY.rst b/HISTORY.rst index 32384a4..6f33531 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,27 @@ Changelog Here's the recent changes to AST Unparser. +1.6.3 - 2019-12-22 +~~~~~~~~~~~~~~~~~~ + +* Add full support for Python 3.8 + +1.6.2 - 2019-01-19 +~~~~~~~~~~~~~~~~~~ + +* Add support for the Constant node in Python 3.8 +* Add tests to the sdist + +1.6.1 - 2018-10-03 +~~~~~~~~~~~~~~~~~~ + +* Fix the roundtripping of very complex f-strings. + +1.6.0 - 2018-09-30 +~~~~~~~~~~~~~~~~~~ + +* Python 3.7 compatibility + 1.5.0 - 2017-02-05 ~~~~~~~~~~~~~~~~~~ diff --git a/MANIFEST.in b/MANIFEST.in index ae65950..60e5939 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,5 @@ include HISTORY.rst include LICENSE include README.rst include requirements.txt -include test_requirements.txt \ No newline at end of file +include test_requirements.txt +recursive-include tests *.py diff --git a/lib/astunparse/__init__.py b/lib/astunparse/__init__.py index 99b96d7..e2eeddc 100644 --- a/lib/astunparse/__init__.py +++ b/lib/astunparse/__init__.py @@ -5,7 +5,7 @@ from .printer import Printer -__version__ = '1.5.0' +__version__ = '1.6.3' def unparse(tree): diff --git a/lib/astunparse/unparser.py b/lib/astunparse/unparser.py index 71a9456..0ef6fd8 100644 --- a/lib/astunparse/unparser.py +++ b/lib/astunparse/unparser.py @@ -29,7 +29,7 @@ class Unparser: output source code for the abstract syntax; original formatting is disregarded. """ - def __init__(self, tree, file=sys.stdout): + def __init__(self, tree, file = sys.stdout): """Unparser(tree, file=sys.stdout) -> None. Print the source for tree to file.""" self.f = file @@ -89,6 +89,13 @@ def _Expr(self, tree): self.fill() self.dispatch(tree.value) + def _NamedExpr(self, tree): + self.write("(") + self.dispatch(tree.target) + self.write(" := ") + self.dispatch(tree.value) + self.write(")") + def _Import(self, t): self.fill("import ") interleave(lambda: self.write(", "), self.dispatch, t.names) @@ -120,11 +127,11 @@ def _AugAssign(self, t): def _AnnAssign(self, t): self.fill() - if not t.simple: - self.write("(") + if not t.simple and isinstance(t.target, ast.Name): + self.write('(') self.dispatch(t.target) - if not t.simple: - self.write(")") + if not t.simple and isinstance(t.target, ast.Name): + self.write(')') self.write(": ") self.dispatch(t.annotation) if t.value: @@ -189,6 +196,14 @@ def _Nonlocal(self, t): self.fill("nonlocal ") interleave(lambda: self.write(", "), self.write, t.names) + def _Await(self, t): + self.write("(") + self.write("await") + if t.value: + self.write(" ") + self.dispatch(t.value) + self.write(")") + def _Yield(self, t): self.write("(") self.write("yield") @@ -328,12 +343,19 @@ def _ClassDef(self, t): self.dispatch(t.body) self.leave() - def _generic_FunctionDef(self, t, async_=False): + def _FunctionDef(self, t): + self.__FunctionDef_helper(t, "def") + + def _AsyncFunctionDef(self, t): + self.__FunctionDef_helper(t, "async def") + + def __FunctionDef_helper(self, t, fill_suffix): self.write("\n") for deco in t.decorator_list: self.fill("@") self.dispatch(deco) - self.fill(("async " if async_ else "") + "def " + t.name + "(") + def_str = fill_suffix+" "+t.name + "(" + self.fill(def_str) self.dispatch(t.args) self.write(")") if getattr(t, "returns", False): @@ -343,14 +365,14 @@ def _generic_FunctionDef(self, t, async_=False): self.dispatch(t.body) self.leave() - def _FunctionDef(self, t): - self._generic_FunctionDef(t) + def _For(self, t): + self.__For_helper("for ", t) - def _AsyncFunctionDef(self, t): - self._generic_FunctionDef(t, async_=True) + def _AsyncFor(self, t): + self.__For_helper("async for ", t) - def _generic_For(self, t, async_=False): - self.fill("async for " if async_ else "for ") + def __For_helper(self, fill, t): + self.fill(fill) self.dispatch(t.target) self.write(" in ") self.dispatch(t.iter) @@ -363,12 +385,6 @@ def _generic_For(self, t, async_=False): self.dispatch(t.orelse) self.leave() - def _For(self, t): - self._generic_For(t) - - def _AsyncFor(self, t): - self._generic_For(t, async_=True) - def _If(self, t): self.fill("if ") self.dispatch(t.test) @@ -442,33 +458,64 @@ def _Str(self, tree): else: assert False, "shouldn't get here" - format_conversions = {97: 'a', 114: 'r', 115: 's'} + def _JoinedStr(self, t): + # JoinedStr(expr* values) + self.write("f") + string = StringIO() + self._fstring_JoinedStr(t, string.write) + # Deviation from `unparse.py`: Try to find an unused quote. + # This change is made to handle _very_ complex f-strings. + v = string.getvalue() + if '\n' in v or '\r' in v: + quote_types = ["'''", '"""'] + else: + quote_types = ["'", '"', '"""', "'''"] + for quote_type in quote_types: + if quote_type not in v: + v = "{quote_type}{v}{quote_type}".format(quote_type=quote_type, v=v) + break + else: + v = repr(v) + self.write(v) def _FormattedValue(self, t): # FormattedValue(expr value, int? conversion, expr? format_spec) - self.write("{") - self.dispatch(t.value) - if t.conversion is not None and t.conversion != -1: - self.write("!") - self.write(self.format_conversions[t.conversion]) - #raise NotImplementedError(ast.dump(t, True, True)) - if t.format_spec is not None: - self.write(":") - if isinstance(t.format_spec, ast.Str): - self.write(t.format_spec.s) - else: - self.dispatch(t.format_spec) - self.write("}") + self.write("f") + string = StringIO() + self._fstring_JoinedStr(t, string.write) + self.write(repr(string.getvalue())) - def _JoinedStr(self, t): - # JoinedStr(expr* values) - self.write("f'''") + def _fstring_JoinedStr(self, t, write): for value in t.values: - if isinstance(value, ast.Str): - self.write(value.s) - else: - self.dispatch(value) - self.write("'''") + meth = getattr(self, "_fstring_" + type(value).__name__) + meth(value, write) + + def _fstring_Str(self, t, write): + value = t.s.replace("{", "{{").replace("}", "}}") + write(value) + + def _fstring_Constant(self, t, write): + assert isinstance(t.value, str) + value = t.value.replace("{", "{{").replace("}", "}}") + write(value) + + def _fstring_FormattedValue(self, t, write): + write("{") + expr = StringIO() + Unparser(t.value, expr) + expr = expr.getvalue().rstrip("\n") + if expr.startswith("{"): + write(" ") # Separate pair of opening brackets as "{ {" + write(expr) + if t.conversion != -1: + conversion = chr(t.conversion) + assert conversion in "sra" + write("!{conversion}".format(conversion=conversion)) + if t.format_spec: + write(":") + meth = getattr(self, "_fstring_" + type(t.format_spec).__name__) + meth(t.format_spec, write) + write("}") def _Name(self, t): self.write(t.id) @@ -481,6 +528,30 @@ def _Repr(self, t): self.dispatch(t.value) self.write("`") + def _write_constant(self, value): + if isinstance(value, (float, complex)): + # Substitute overflowing decimal literal for AST infinities. + self.write(repr(value).replace("inf", INFSTR)) + else: + self.write(repr(value)) + + def _Constant(self, t): + value = t.value + if isinstance(value, tuple): + self.write("(") + if len(value) == 1: + self._write_constant(value[0]) + self.write(",") + else: + interleave(lambda: self.write(", "), self._write_constant, value) + self.write(")") + elif value is Ellipsis: # instead of `...` for Py2 compatibility + self.write("...") + else: + if t.kind == "u": + self.write("u") + self._write_constant(t.value) + def _Num(self, t): repr_n = repr(t.n) if six.PY3: @@ -533,8 +604,9 @@ def _DictComp(self, t): def _comprehension(self, t): if getattr(t, 'is_async', False): - self.write(" async") - self.write(" for ") + self.write(" async for ") + else: + self.write(" for ") self.dispatch(t.target) self.write(" in ") self.dispatch(t.iter) @@ -559,22 +631,27 @@ def _Set(self, t): def _Dict(self, t): self.write("{") - def write_pair(pair): - (k, v) = pair + def write_key_value_pair(k, v): self.dispatch(k) self.write(": ") self.dispatch(v) - self.write(",") - self._indent +=1 - self.fill("") - interleave(lambda: self.fill(""), write_pair, zip(t.keys, t.values)) - self._indent -=1 - self.fill("}") + + def write_item(item): + k, v = item + if k is None: + # for dictionary unpacking operator in dicts {**{'y': 2}} + # see PEP 448 for details + self.write("**") + self.dispatch(v) + else: + write_key_value_pair(k, v) + interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values)) + self.write("}") def _Tuple(self, t): self.write("(") if len(t.elts) == 1: - (elt,) = t.elts + elt = t.elts[0] self.dispatch(elt) self.write(",") else: @@ -599,10 +676,9 @@ def _UnaryOp(self, t): self.dispatch(t.operand) self.write(")") - binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%", + binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%", "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", - "FloorDiv":"//", "Pow": "**", - "MatMult":"@"} + "FloorDiv":"//", "Pow": "**"} def _BinOp(self, t): self.write("(") self.dispatch(t.left) @@ -632,7 +708,7 @@ def _Attribute(self,t): # Special case: 3.__abs__() is a syntax error, so if t.value # is an integer literal then we need to either parenthesize # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): + if isinstance(t.value, getattr(ast, 'Constant', getattr(ast, 'Num', None))) and isinstance(t.value.n, int): self.write(" ") self.write(".") self.write(t.attr) @@ -703,18 +779,22 @@ def _arg(self, t): def _arguments(self, t): first = True # normal arguments - defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults - for a,d in zip(t.args, defaults): + all_args = getattr(t, 'posonlyargs', []) + t.args + defaults = [None] * (len(all_args) - len(t.defaults)) + t.defaults + for index, elements in enumerate(zip(all_args, defaults), 1): + a, d = elements if first:first = False else: self.write(", ") self.dispatch(a) if d: self.write("=") self.dispatch(d) + if index == len(getattr(t, 'posonlyargs', ())): + self.write(", /") # varargs, or bare '*' if no varargs but keyword-only arguments present if t.vararg or getattr(t, "kwonlyargs", False): - if first: first = False + if first:first = False else: self.write(", ") self.write("*") if t.vararg: @@ -782,14 +862,6 @@ def _withitem(self, t): self.write(" as ") self.dispatch(t.optional_vars) - def _Await(self, t): - self.write("(") - self.write("await") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - def roundtrip(filename, output=sys.stdout): if six.PY3: with open(filename, "rb") as pyfile: diff --git a/setup.py b/setup.py index 049939f..e5a277a 100755 --- a/setup.py +++ b/setup.py @@ -48,10 +48,10 @@ def read_version(): "Programming Language :: Python :: 2", 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Software Development :: Code Generators', ], test_suite='tests', diff --git a/tests/common.py b/tests/common.py index f930308..95b9755 100644 --- a/tests/common.py +++ b/tests/common.py @@ -139,6 +139,10 @@ class Foo: pass `{}` """ +complex_f_string = '''\ +f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\' +''' + async_function_def = """\ async def f(): suite1 @@ -211,6 +215,7 @@ def test_unary_parens(self): self.check_roundtrip("not True or False") self.check_roundtrip("True or not False") + @unittest.skipUnless(sys.version_info < (3, 6), "Only works for Python < 3.6") def test_integer_parens(self): self.check_roundtrip("3 .__abs__()") @@ -288,11 +293,21 @@ def test_raise_from(self): def test_bytes(self): self.check_roundtrip("b'123'") + @unittest.skipIf(sys.version_info < (3, 6), "Not supported < 3.6") + def test_formatted_value(self): + self.check_roundtrip('f"{value}"') + self.check_roundtrip('f"{value!s}"') + self.check_roundtrip('f"{value:4}"') + self.check_roundtrip('f"{value!s:4}"') + @unittest.skipIf(sys.version_info < (3, 6), "Not supported < 3.6") def test_joined_str(self): self.check_roundtrip('f"{key}={value!s}"') self.check_roundtrip('f"{key}={value!r}"') self.check_roundtrip('f"{key}={value!a}"') + + @unittest.skipIf(sys.version_info != (3, 6, 0), "Only supported on 3.6.0") + def test_joined_str_361(self): self.check_roundtrip('f"{key:4}={value!s}"') self.check_roundtrip('f"{key:02}={value!r}"') self.check_roundtrip('f"{key:6}={value!a}"') @@ -307,6 +322,10 @@ def test_joined_str(self): def test_repr(self): self.check_roundtrip(a_repr) + @unittest.skipUnless(sys.version_info[:2] >= (3, 6), "Only for Python 3.6 or greater") + def test_complex_f_string(self): + self.check_roundtrip(complex_f_string) + @unittest.skipUnless(six.PY3, "Only for Python 3") def test_annotations(self): self.check_roundtrip("def f(a : int): pass") @@ -327,6 +346,11 @@ def test_set_comprehension(self): def test_dict_comprehension(self): self.check_roundtrip("{x: x*x for x in range(10)}") + @unittest.skipIf(sys.version_info < (3, 6), "Not supported < 3.6") + def test_dict_with_unpacking(self): + self.check_roundtrip("{**x}") + self.check_roundtrip("{a: b, **x}") + @unittest.skipIf(sys.version_info < (3, 6), "Not supported < 3.6") def test_async_comp_and_gen_in_async_function(self): self.check_roundtrip(async_comprehensions_and_generators) diff --git a/tox.ini b/tox.ini index 635ead5..f6953b6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py32, py33, py34, py35 +envlist = py27, py35, py36, py37, py38 [testenv] usedevelop = True