diff --git a/babel/messages/extract.py b/babel/messages/extract.py index 8d4bbeaf8..975ec7c32 100644 --- a/babel/messages/extract.py +++ b/babel/messages/extract.py @@ -36,6 +36,8 @@ from tokenize import COMMENT, NAME, OP, STRING, generate_tokens from typing import TYPE_CHECKING, Any +from more_itertools import peekable + from babel.messages._compat import find_entrypoints from babel.util import parse_encoding, parse_future_flags, pathmatch @@ -514,12 +516,35 @@ def extract_python( future_flags = parse_future_flags(fileobj, encoding) next_line = lambda: fileobj.readline().decode(encoding) - tokens = generate_tokens(next_line) + tokens = peekable(generate_tokens(next_line)) # Current prefix of a Python 3.12 (PEP 701) f-string, or None if we're not # currently parsing one. current_fstring_start = None + def peek_dotted_name(name) -> str: + """Used to peek ahead in the token stream to get a dotted name, if possible""" + + nonlocal tokens + + prev_tok = NAME + + while True: + tok, value, *_ = tokens.peek() + + if prev_tok == NAME and tok == OP and value == '.': + name += '.' + next(tokens) + elif tok == NAME: + name += value + next(tokens) + else: + break + + prev_tok = tok + + return name + for tok, value, (lineno, _), _, _ in tokens: if call_stack == -1 and tok == NAME and value in ('def', 'class'): in_def = True @@ -552,7 +577,12 @@ def extract_python( translator_comments.append((lineno, value)) break elif funcname and call_stack == 0: - nested = (tok == NAME and value in keywords) + if tok == NAME: + maybe_funcname: str | None = peek_dotted_name(value) + else: + maybe_funcname = None + + nested = (maybe_funcname and maybe_funcname in keywords) if (tok == OP and value == ')') or nested: if buf: messages.append(''.join(buf)) @@ -576,7 +606,7 @@ def extract_python( translator_comments = [] in_translator_comments = False if nested: - funcname = value + funcname = maybe_funcname elif tok == STRING: val = _parse_python_string(value, encoding, future_flags) if val is not None: @@ -612,8 +642,11 @@ def extract_python( call_stack -= 1 elif funcname and call_stack == -1: funcname = None - elif tok == NAME and value in keywords: - funcname = value + elif tok == NAME: + peeked_name = peek_dotted_name(value) + + if peeked_name in keywords: + funcname = peeked_name if (current_fstring_start is not None and tok not in {FSTRING_START, FSTRING_MIDDLE} diff --git a/setup.py b/setup.py index 23936a75d..3ece7ebf4 100755 --- a/setup.py +++ b/setup.py @@ -66,6 +66,7 @@ def run(self): # higher. # Python 3.9 and later include zoneinfo which replaces pytz 'pytz>=2015.7; python_version<"3.9"', + 'more-itertools' ], extras_require={ 'dev': [ diff --git a/tests/messages/test_extract.py b/tests/messages/test_extract.py index 7d3a05aa7..d5fbfc934 100644 --- a/tests/messages/test_extract.py +++ b/tests/messages/test_extract.py @@ -441,6 +441,19 @@ def test_nested_messages(self): assert messages[7][2] == 'Armin' assert messages[7][3] == [] + def test_dotted_keywords(self): + buf = BytesIO(b"""\ +msg0 = self.gettext('Hello there') +msg1 = self.gettext('Thanks {user}', name=self.gettext('User')) +msg3 = self.gettext('Can use {both}', both=gettext('gettexts')) +""") + messages = list(extract.extract_python(buf, ('self.gettext', 'gettext'), [], {})) + assert messages[0] == (1, 'self.gettext', 'Hello there', []) + assert messages[1] == (2, 'self.gettext', ('Thanks {user}', None), []) + assert messages[2] == (2, 'self.gettext', 'User', []) + assert messages[3] == (3, 'self.gettext', ('Can use {both}', None), []) + assert messages[4] == (3, 'gettext', 'gettexts', []) + class ExtractTestCase(unittest.TestCase):