From bda29a7089c3a21e9e49887f54cd73e38f6445d5 Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Mon, 18 Jan 2016 16:54:23 +0200
Subject: [PATCH 1/6] JavaScript: modernize lexer slightly

---
 babel/messages/jslexer.py | 37 +++++++++++++------------------------
 1 file changed, 13 insertions(+), 24 deletions(-)

diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index 22c6e1f9c..282d294f3 100644
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -9,27 +9,34 @@
     :copyright: (c) 2013 by the Babel Team.
     :license: BSD, see LICENSE for more details.
 """
-
-from operator import itemgetter
+from collections import namedtuple
 import re
 from babel._compat import unichr
 
-operators = [
+operators = sorted([
     '+', '-', '*', '%', '!=', '==', '<', '>', '<=', '>=', '=',
     '+=', '-=', '*=', '%=', '<<', '>>', '>>>', '<<=', '>>=',
     '>>>=', '&', '&=', '|', '|=', '&&', '||', '^', '^=', '(', ')',
     '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.', ':'
-]
-operators.sort(key=lambda a: -len(a))
+], key=len, reverse=True)
 
 escapes = {'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t'}
 
+division_re = re.compile(r'/=?')
+regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*(?s)')
+line_re = re.compile(r'(\r\n|\n|\r)')
+line_join_re = re.compile(r'\\' + line_re.pattern)
+uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
+name_re = re.compile(r'(\$+\w*|[^\W\d]\w*)(?u)')
+
+Token = namedtuple('Token', 'type value lineno')
+
 rules = [
     (None, re.compile(r'\s+(?u)')),
     (None, re.compile(r'<!--.*')),
     ('linecomment', re.compile(r'//.*')),
     ('multilinecomment', re.compile(r'/\*.*?\*/(?us)')),
-    ('name', re.compile(r'(\$+\w*|[^\W\d]\w*)(?u)')),
+    ('name', name_re),
     ('number', re.compile(r'''(?x)(
         (?:0|[1-9]\d*)
         (\.\d+)?
@@ -43,24 +50,6 @@
     )'''))
 ]
 
-division_re = re.compile(r'/=?')
-regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*(?s)')
-line_re = re.compile(r'(\r\n|\n|\r)')
-line_join_re = re.compile(r'\\' + line_re.pattern)
-uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
-
-
-class Token(tuple):
-    """Represents a token as returned by `tokenize`."""
-    __slots__ = ()
-
-    def __new__(cls, type, value, lineno):
-        return tuple.__new__(cls, (type, value, lineno))
-
-    type = property(itemgetter(0))
-    value = property(itemgetter(1))
-    lineno = property(itemgetter(2))
-
 
 def indicates_division(token):
     """A helper function that helps the tokenizer to decide if the current

From d4cafa495097d86ae9ff46c8be9273274a7b5987 Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Wed, 30 Dec 2015 19:29:48 +0200
Subject: [PATCH 2/6] JavaScript: Move extraction test cases into separate file

And remove unittest boilerplate while at it
---
 tests/messages/test_extract.py    | 91 ----------------------------
 tests/messages/test_js_extract.py | 99 +++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+), 91 deletions(-)
 create mode 100644 tests/messages/test_js_extract.py

diff --git a/tests/messages/test_extract.py b/tests/messages/test_extract.py
index 66c8cd908..2eee88ed9 100644
--- a/tests/messages/test_extract.py
+++ b/tests/messages/test_extract.py
@@ -410,97 +410,6 @@ def test_extract_strip_comment_tags(self):
                           u'a prefix too'], messages[1][2])
 
 
-class ExtractJavaScriptTestCase(unittest.TestCase):
-
-    def test_simple_extract(self):
-        buf = BytesIO(b"""\
-msg1 = _('simple')
-msg2 = gettext('simple')
-msg3 = ngettext('s', 'p', 42)
-        """)
-        messages = \
-            list(extract.extract('javascript', buf, extract.DEFAULT_KEYWORDS,
-                                 [], {}))
-
-        self.assertEqual([(1, 'simple', [], None),
-                          (2, 'simple', [], None),
-                          (3, ('s', 'p'), [], None)], messages)
-
-    def test_various_calls(self):
-        buf = BytesIO(b"""\
-msg1 = _(i18n_arg.replace(/"/, '"'))
-msg2 = ungettext(i18n_arg.replace(/"/, '"'), multi_arg.replace(/"/, '"'), 2)
-msg3 = ungettext("Babel", multi_arg.replace(/"/, '"'), 2)
-msg4 = ungettext(i18n_arg.replace(/"/, '"'), "Babels", 2)
-msg5 = ungettext('bunny', 'bunnies', parseInt(Math.random() * 2 + 1))
-msg6 = ungettext(arg0, 'bunnies', rparseInt(Math.random() * 2 + 1))
-msg7 = _(hello.there)
-msg8 = gettext('Rabbit')
-msg9 = dgettext('wiki', model.addPage())
-msg10 = dngettext(domain, 'Page', 'Pages', 3)
-""")
-        messages = \
-            list(extract.extract('javascript', buf, extract.DEFAULT_KEYWORDS, [],
-                                 {}))
-        self.assertEqual([(5, (u'bunny', u'bunnies'), [], None),
-                          (8, u'Rabbit', [], None),
-                          (10, (u'Page', u'Pages'), [], None)], messages)
-
-    def test_message_with_line_comment(self):
-        buf = BytesIO(u"""\
-// NOTE: hello
-msg = _('Bonjour à tous')
-""".encode('utf-8'))
-        messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
-        self.assertEqual(u'Bonjour à tous', messages[0][2])
-        self.assertEqual([u'NOTE: hello'], messages[0][3])
-
-    def test_message_with_multiline_comment(self):
-        buf = BytesIO(u"""\
-/* NOTE: hello
-   and bonjour
-     and servus */
-msg = _('Bonjour à tous')
-""".encode('utf-8'))
-        messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
-        self.assertEqual(u'Bonjour à tous', messages[0][2])
-        self.assertEqual([u'NOTE: hello', 'and bonjour', '  and servus'], messages[0][3])
-
-    def test_ignore_function_definitions(self):
-        buf = BytesIO(b"""\
-function gettext(value) {
-    return translations[language][value] || value;
-}""")
-
-        messages = list(extract.extract_javascript(buf, ('gettext',), [], {}))
-        self.assertEqual(messages, [])
-
-    def test_misplaced_comments(self):
-        buf = BytesIO(b"""\
-/* NOTE: this won't show up */
-foo()
-
-/* NOTE: this will */
-msg = _('Something')
-
-// NOTE: this will show up
-// too.
-msg = _('Something else')
-
-// NOTE: but this won't
-bar()
-
-_('no comment here')
-""")
-        messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
-        self.assertEqual(u'Something', messages[0][2])
-        self.assertEqual([u'NOTE: this will'], messages[0][3])
-        self.assertEqual(u'Something else', messages[1][2])
-        self.assertEqual([u'NOTE: this will show up', 'too.'], messages[1][3])
-        self.assertEqual(u'no comment here', messages[2][2])
-        self.assertEqual([], messages[2][3])
-
-
 class ExtractTestCase(unittest.TestCase):
 
     def test_invalid_filter(self):
diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py
new file mode 100644
index 000000000..299240e21
--- /dev/null
+++ b/tests/messages/test_js_extract.py
@@ -0,0 +1,99 @@
+# -- encoding: UTF-8 --
+from babel._compat import BytesIO
+from babel.messages import extract
+
+
+def test_simple_extract():
+    buf = BytesIO(b"""\
+msg1 = _('simple')
+msg2 = gettext('simple')
+msg3 = ngettext('s', 'p', 42)
+    """)
+    messages = \
+        list(extract.extract('javascript', buf, extract.DEFAULT_KEYWORDS,
+                             [], {}))
+
+    assert messages == [(1, 'simple', [], None),
+                        (2, 'simple', [], None),
+                        (3, ('s', 'p'), [], None)]
+
+
+def test_various_calls():
+    buf = BytesIO(b"""\
+msg1 = _(i18n_arg.replace(/"/, '"'))
+msg2 = ungettext(i18n_arg.replace(/"/, '"'), multi_arg.replace(/"/, '"'), 2)
+msg3 = ungettext("Babel", multi_arg.replace(/"/, '"'), 2)
+msg4 = ungettext(i18n_arg.replace(/"/, '"'), "Babels", 2)
+msg5 = ungettext('bunny', 'bunnies', parseInt(Math.random() * 2 + 1))
+msg6 = ungettext(arg0, 'bunnies', rparseInt(Math.random() * 2 + 1))
+msg7 = _(hello.there)
+msg8 = gettext('Rabbit')
+msg9 = dgettext('wiki', model.addPage())
+msg10 = dngettext(domain, 'Page', 'Pages', 3)
+""")
+    messages = \
+        list(extract.extract('javascript', buf, extract.DEFAULT_KEYWORDS, [],
+                             {}))
+    assert messages == [
+        (5, (u'bunny', u'bunnies'), [], None),
+        (8, u'Rabbit', [], None),
+        (10, (u'Page', u'Pages'), [], None)
+    ]
+
+
+def test_message_with_line_comment():
+    buf = BytesIO(u"""\
+// NOTE: hello
+msg = _('Bonjour à tous')
+""".encode('utf-8'))
+    messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
+    assert messages[0][2] == u'Bonjour à tous'
+    assert messages[0][3] == [u'NOTE: hello']
+
+
+def test_message_with_multiline_comment():
+    buf = BytesIO(u"""\
+/* NOTE: hello
+and bonjour
+  and servus */
+msg = _('Bonjour à tous')
+""".encode('utf-8'))
+    messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
+    assert messages[0][2] == u'Bonjour à tous'
+    assert messages[0][3] == [u'NOTE: hello', 'and bonjour', '  and servus']
+
+
+def test_ignore_function_definitions():
+    buf = BytesIO(b"""\
+function gettext(value) {
+return translations[language][value] || value;
+}""")
+
+    messages = list(extract.extract_javascript(buf, ('gettext',), [], {}))
+    assert not messages
+
+
+def test_misplaced_comments():
+    buf = BytesIO(b"""\
+/* NOTE: this won't show up */
+foo()
+
+/* NOTE: this will */
+msg = _('Something')
+
+// NOTE: this will show up
+// too.
+msg = _('Something else')
+
+// NOTE: but this won't
+bar()
+
+_('no comment here')
+""")
+    messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
+    assert messages[0][2] == u'Something'
+    assert messages[0][3] == [u'NOTE: this will']
+    assert messages[1][2] == u'Something else'
+    assert messages[1][3] == [u'NOTE: this will show up', 'too.']
+    assert messages[2][2] == u'no comment here'
+    assert messages[2][3] == []

From 8fee76cf39ee4192da534fbafde6ef2e90420429 Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Mon, 18 Jan 2016 17:09:08 +0200
Subject: [PATCH 3/6] JavaScript: Add JSX/E4X tag lexing

Fixes #280
---
 babel/messages/extract.py         |  4 +++-
 babel/messages/jslexer.py         | 25 ++++++++++++++++++++++---
 tests/messages/test_js_extract.py | 25 +++++++++++++++++++++++++
 3 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/babel/messages/extract.py b/babel/messages/extract.py
index c2dcd5b99..153fe5116 100644
--- a/babel/messages/extract.py
+++ b/babel/messages/extract.py
@@ -506,6 +506,8 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
     :param comment_tags: a list of translator tags to search for and include
                          in the results
     :param options: a dictionary of additional options (optional)
+                    Supported options are:
+                    * `jsx` -- set to false to disable JSX/E4X support.
     """
     from babel.messages.jslexer import tokenize, unquote_string
     funcname = message_lineno = None
@@ -517,7 +519,7 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
     last_token = None
     call_stack = -1
 
-    for token in tokenize(fileobj.read().decode(encoding)):
+    for token in tokenize(fileobj.read().decode(encoding), jsx=options.get("jsx", True)):
         if token.type == 'operator' and token.value == '(':
             if funcname:
                 message_lineno = token.lineno
diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index 282d294f3..32500676a 100644
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -31,7 +31,7 @@
 
 Token = namedtuple('Token', 'type value lineno')
 
-rules = [
+_rules = [
     (None, re.compile(r'\s+(?u)')),
     (None, re.compile(r'<!--.*')),
     ('linecomment', re.compile(r'//.*')),
@@ -43,6 +43,7 @@
         ([eE][-+]?\d+)? |
         (0x[a-fA-F0-9]+)
     )''')),
+    ('jsx_tag', re.compile(r'<(?:/?)\w+.+?>', re.I)),  # May be mangled in `get_rules`
     ('operator', re.compile(r'(%s)' % '|'.join(map(re.escape, operators)))),
     ('string', re.compile(r'''(?xs)(
         '(?:[^'\\]*(?:\\.[^'\\]*)*)'  |
@@ -51,6 +52,20 @@
 ]
 
 
+def get_rules(jsx):
+    """
+    Get a tokenization rule list given the passed syntax options.
+
+    Internal to this module.
+    """
+    rules = []
+    for token_type, rule in _rules:
+        if not jsx and token_type and 'jsx' in token_type:
+            continue
+        rules.append((token_type, rule))
+    return rules
+
+
 def indicates_division(token):
     """A helper function that helps the tokenizer to decide if the current
     token may be followed by a division operator.
@@ -116,13 +131,17 @@ def unquote_string(string):
     return u''.join(result)
 
 
-def tokenize(source):
-    """Tokenize a JavaScript source.  Returns a generator of tokens.
+def tokenize(source, jsx=True):
+    """
+    Tokenize JavaScript/JSX source.  Returns a generator of tokens.
+
+    :param jsx: Enable (limited) JSX parsing.
     """
     may_divide = False
     pos = 0
     lineno = 1
     end = len(source)
+    rules = get_rules(jsx=jsx)
 
     while pos < end:
         # handle regular rules first
diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py
index 299240e21..ae6d277b3 100644
--- a/tests/messages/test_js_extract.py
+++ b/tests/messages/test_js_extract.py
@@ -1,4 +1,5 @@
 # -- encoding: UTF-8 --
+import pytest
 from babel._compat import BytesIO
 from babel.messages import extract
 
@@ -97,3 +98,27 @@ def test_misplaced_comments():
     assert messages[1][3] == [u'NOTE: this will show up', 'too.']
     assert messages[2][2] == u'no comment here'
     assert messages[2][3] == []
+
+
+JSX_SOURCE = b"""
+class Foo {
+    render() {
+        const value = gettext("hello");
+        return (
+            <option value="val1">{ i18n._('String1') }</option>
+            <option value="val2">{ i18n._('String 2') }</option>
+            <option value="val3">{ i18n._('String 3') }</option>
+        );
+    }
+"""
+EXPECTED_JSX_MESSAGES = ["hello", "String1", "String 2", "String 3"]
+
+
+@pytest.mark.parametrize("jsx_enabled", (False, True))
+def test_jsx_extraction(jsx_enabled):
+    buf = BytesIO(JSX_SOURCE)
+    messages = [m[2] for m in extract.extract_javascript(buf, ('_', 'gettext'), [], {"jsx": jsx_enabled})]
+    if jsx_enabled:
+        assert messages == EXPECTED_JSX_MESSAGES
+    else:
+        assert messages != EXPECTED_JSX_MESSAGES

From d0c3853728cf242cfd1b637d17a685a4f1864d73 Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Mon, 18 Jan 2016 17:09:57 +0200
Subject: [PATCH 4/6] JavaScript: Support dotted names as keywords

Fixes #148
---
 babel/messages/extract.py         |  7 ++++++-
 babel/messages/jslexer.py         | 15 +++++++++++----
 tests/messages/test_js_extract.py |  9 +++++++++
 tests/messages/test_jslexer.py    | 19 +++++++++++++++++++
 4 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/babel/messages/extract.py b/babel/messages/extract.py
index 153fe5116..ee90afb76 100644
--- a/babel/messages/extract.py
+++ b/babel/messages/extract.py
@@ -518,8 +518,13 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
     encoding = options.get('encoding', 'utf-8')
     last_token = None
     call_stack = -1
+    dotted = any('.' in kw for kw in keywords)
 
-    for token in tokenize(fileobj.read().decode(encoding), jsx=options.get("jsx", True)):
+    for token in tokenize(
+        fileobj.read().decode(encoding),
+        jsx=options.get("jsx", True),
+        dotted=dotted
+    ):
         if token.type == 'operator' and token.value == '(':
             if funcname:
                 message_lineno = token.lineno
diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index 32500676a..e6310cf3f 100644
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -22,12 +22,13 @@
 
 escapes = {'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t'}
 
+name_re = re.compile(r'[\w$_][\w\d$_]*', re.UNICODE)
+dotted_name_re = re.compile(r'[\w$_][\w\d$_.]*[\w\d$_.]', re.UNICODE)
 division_re = re.compile(r'/=?')
 regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*(?s)')
 line_re = re.compile(r'(\r\n|\n|\r)')
 line_join_re = re.compile(r'\\' + line_re.pattern)
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
-name_re = re.compile(r'(\$+\w*|[^\W\d]\w*)(?u)')
 
 Token = namedtuple('Token', 'type value lineno')
 
@@ -36,6 +37,7 @@
     (None, re.compile(r'<!--.*')),
     ('linecomment', re.compile(r'//.*')),
     ('multilinecomment', re.compile(r'/\*.*?\*/(?us)')),
+    ('dotted_name', dotted_name_re),
     ('name', name_re),
     ('number', re.compile(r'''(?x)(
         (?:0|[1-9]\d*)
@@ -52,7 +54,7 @@
 ]
 
 
-def get_rules(jsx):
+def get_rules(jsx, dotted):
     """
     Get a tokenization rule list given the passed syntax options.
 
@@ -62,6 +64,10 @@ def get_rules(jsx):
     for token_type, rule in _rules:
         if not jsx and token_type and 'jsx' in token_type:
             continue
+        if token_type == 'dotted_name':
+            if not dotted:
+                continue
+            token_type = 'name'
         rules.append((token_type, rule))
     return rules
 
@@ -131,17 +137,18 @@ def unquote_string(string):
     return u''.join(result)
 
 
-def tokenize(source, jsx=True):
+def tokenize(source, jsx=True, dotted=True):
     """
     Tokenize JavaScript/JSX source.  Returns a generator of tokens.
 
     :param jsx: Enable (limited) JSX parsing.
+    :param dotted: Read dotted names as single name token.
     """
     may_divide = False
     pos = 0
     lineno = 1
     end = len(source)
-    rules = get_rules(jsx=jsx)
+    rules = get_rules(jsx=jsx, dotted=dotted)
 
     while pos < end:
         # handle regular rules first
diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py
index ae6d277b3..203bf4cba 100644
--- a/tests/messages/test_js_extract.py
+++ b/tests/messages/test_js_extract.py
@@ -122,3 +122,12 @@ def test_jsx_extraction(jsx_enabled):
         assert messages == EXPECTED_JSX_MESSAGES
     else:
         assert messages != EXPECTED_JSX_MESSAGES
+
+
+def test_dotted_keyword_extract():
+    buf = BytesIO(b"msg1 = com.corporate.i18n.formatMessage('Insert coin to continue')")
+    messages = list(
+        extract.extract('javascript', buf, {"com.corporate.i18n.formatMessage": None}, [], {})
+    )
+
+    assert messages == [(1, 'Insert coin to continue', [], None)]
diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py
index 4ffb4a9e4..7277104f2 100644
--- a/tests/messages/test_jslexer.py
+++ b/tests/messages/test_jslexer.py
@@ -6,3 +6,22 @@
 def test_unquote():
     assert jslexer.unquote_string('""') == ''
     assert jslexer.unquote_string(r'"h\u00ebllo"') == u"hëllo"
+
+
+def test_dollar_in_identifier():
+    assert list(jslexer.tokenize('dollar$dollar')) == [('name', 'dollar$dollar', 1)]
+
+
+def test_dotted_name():
+    assert list(jslexer.tokenize("foo.bar(quux)", dotted=True)) == [
+        ('name', 'foo.bar', 1),
+        ('operator', '(', 1),
+        ('name', 'quux', 1),
+        ('operator', ')', 1)
+    ]
+
+
+def test_dotted_name_end():
+    assert list(jslexer.tokenize("foo.bar", dotted=True)) == [
+        ('name', 'foo.bar', 1),
+    ]

From f061cc16415dc0399f7fcea62012617885ab495b Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Mon, 18 Jan 2016 17:35:05 +0200
Subject: [PATCH 5/6] JavaScript: add basic support for template strings

Refs #329
---
 babel/messages/extract.py         |  5 ++++-
 babel/messages/jslexer.py         | 14 +++++++++-----
 tests/messages/test_js_extract.py |  9 +++++++++
 tests/messages/test_jslexer.py    |  7 +++++++
 4 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/babel/messages/extract.py b/babel/messages/extract.py
index ee90afb76..2e5053759 100644
--- a/babel/messages/extract.py
+++ b/babel/messages/extract.py
@@ -508,6 +508,8 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
     :param options: a dictionary of additional options (optional)
                     Supported options are:
                     * `jsx` -- set to false to disable JSX/E4X support.
+                    * `template_string` -- set to false to disable ES6
+                                           template string support.
     """
     from babel.messages.jslexer import tokenize, unquote_string
     funcname = message_lineno = None
@@ -523,6 +525,7 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
     for token in tokenize(
         fileobj.read().decode(encoding),
         jsx=options.get("jsx", True),
+        template_string=options.get("template_string", True),
         dotted=dotted
     ):
         if token.type == 'operator' and token.value == '(':
@@ -584,7 +587,7 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
                 messages = []
                 call_stack = -1
 
-            elif token.type == 'string':
+            elif token.type in ('string', 'template_string'):
                 new_value = unquote_string(token.value)
                 if concatenate_next:
                     last_argument = (last_argument or '') + new_value
diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index e6310cf3f..aed39f334 100644
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -47,6 +47,7 @@
     )''')),
     ('jsx_tag', re.compile(r'<(?:/?)\w+.+?>', re.I)),  # May be mangled in `get_rules`
     ('operator', re.compile(r'(%s)' % '|'.join(map(re.escape, operators)))),
+    ('template_string', re.compile(r'''`(?:[^`\\]*(?:\\.[^`\\]*)*)`''', re.UNICODE)),
     ('string', re.compile(r'''(?xs)(
         '(?:[^'\\]*(?:\\.[^'\\]*)*)'  |
         "(?:[^"\\]*(?:\\.[^"\\]*)*)"
@@ -54,7 +55,7 @@
 ]
 
 
-def get_rules(jsx, dotted):
+def get_rules(jsx, dotted, template_string):
     """
     Get a tokenization rule list given the passed syntax options.
 
@@ -64,6 +65,8 @@ def get_rules(jsx, dotted):
     for token_type, rule in _rules:
         if not jsx and token_type and 'jsx' in token_type:
             continue
+        if not template_string and token_type == 'template_string':
+            continue
         if token_type == 'dotted_name':
             if not dotted:
                 continue
@@ -83,9 +86,9 @@ def indicates_division(token):
 
 def unquote_string(string):
     """Unquote a string with JavaScript rules.  The string has to start with
-    string delimiters (``'`` or ``"``.)
+    string delimiters (``'``, ``"`` or the back-tick/grave accent (for template strings).)
     """
-    assert string and string[0] == string[-1] and string[0] in '"\'', \
+    assert string and string[0] == string[-1] and string[0] in '"\'`', \
         'string provided is not properly delimited'
     string = line_join_re.sub('\\1', string[1:-1])
     result = []
@@ -137,18 +140,19 @@ def unquote_string(string):
     return u''.join(result)
 
 
-def tokenize(source, jsx=True, dotted=True):
+def tokenize(source, jsx=True, dotted=True, template_string=True):
     """
     Tokenize JavaScript/JSX source.  Returns a generator of tokens.
 
     :param jsx: Enable (limited) JSX parsing.
     :param dotted: Read dotted names as single name token.
+    :param template_string: Support ES6 template strings
     """
     may_divide = False
     pos = 0
     lineno = 1
     end = len(source)
-    rules = get_rules(jsx=jsx, dotted=dotted)
+    rules = get_rules(jsx=jsx, dotted=dotted, template_string=template_string)
 
     while pos < end:
         # handle regular rules first
diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py
index 203bf4cba..f81931900 100644
--- a/tests/messages/test_js_extract.py
+++ b/tests/messages/test_js_extract.py
@@ -131,3 +131,12 @@ def test_dotted_keyword_extract():
     )
 
     assert messages == [(1, 'Insert coin to continue', [], None)]
+
+
+def test_template_string_standard_usage():
+    buf = BytesIO(b"msg1 = gettext(`Very template, wow`)")
+    messages = list(
+        extract.extract('javascript', buf, {"gettext": None}, [], {})
+    )
+
+    assert messages == [(1, 'Very template, wow', [], None)]
diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py
index 7277104f2..84221a1d2 100644
--- a/tests/messages/test_jslexer.py
+++ b/tests/messages/test_jslexer.py
@@ -25,3 +25,10 @@ def test_dotted_name_end():
     assert list(jslexer.tokenize("foo.bar", dotted=True)) == [
         ('name', 'foo.bar', 1),
     ]
+
+
+def test_template_string():
+    assert list(jslexer.tokenize("gettext `foo\"bar\"p`", template_string=True)) == [
+        ('name', 'gettext', 1),
+        ('template_string', '`foo"bar"p`', 1)
+    ]

From 5b09b64982452bea56da65674700d0928ddfecc6 Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Mon, 18 Jan 2016 19:07:17 +0200
Subject: [PATCH 6/6] JavaScript: support tags as keywords for template strings

Refs #329
---
 babel/messages/extract.py         | 12 +++++++++++-
 tests/messages/test_js_extract.py |  9 +++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/babel/messages/extract.py b/babel/messages/extract.py
index 2e5053759..3706df84d 100644
--- a/babel/messages/extract.py
+++ b/babel/messages/extract.py
@@ -511,7 +511,7 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
                     * `template_string` -- set to false to disable ES6
                                            template string support.
     """
-    from babel.messages.jslexer import tokenize, unquote_string
+    from babel.messages.jslexer import Token, tokenize, unquote_string
     funcname = message_lineno = None
     messages = []
     last_argument = None
@@ -528,6 +528,16 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
         template_string=options.get("template_string", True),
         dotted=dotted
     ):
+        if (  # Turn keyword`foo` expressions into keyword("foo") calls:
+            funcname and  # have a keyword...
+            (last_token and last_token.type == 'name') and  # we've seen nothing after the keyword...
+            token.type == 'template_string'  # this is a template string
+        ):
+            message_lineno = token.lineno
+            messages = [unquote_string(token.value)]
+            call_stack = 0
+            token = Token('operator', ')', token.lineno)
+
         if token.type == 'operator' and token.value == '(':
             if funcname:
                 message_lineno = token.lineno
diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py
index f81931900..35322ea16 100644
--- a/tests/messages/test_js_extract.py
+++ b/tests/messages/test_js_extract.py
@@ -140,3 +140,12 @@ def test_template_string_standard_usage():
     )
 
     assert messages == [(1, 'Very template, wow', [], None)]
+
+
+def test_template_string_tag_usage():
+    buf = BytesIO(b"function() { if(foo) msg1 = i18n`Tag template, wow`; }")
+    messages = list(
+        extract.extract('javascript', buf, {"i18n": None}, [], {})
+    )
+
+    assert messages == [(1, 'Tag template, wow', [], None)]