diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py index 6c1680ec..63d8d76d 100644 --- a/sqlparse/filters/others.py +++ b/sqlparse/filters/others.py @@ -74,8 +74,8 @@ def _get_insert_token(token): tidx, token = get_next_comment(idx=tidx) def process(self, stmt): - [self.process(sgroup) for sgroup in stmt.get_sublists()] StripCommentsFilter._process(stmt) + [self.process(sgroup) for sgroup in stmt.get_sublists()] return stmt diff --git a/sqlparse/sql.py b/sqlparse/sql.py index be74694c..a0ce96d5 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -36,27 +36,17 @@ def get_alias(self): return self._get_first_name(reverse=True) -class Token: - """Base class for all other classes in this module. +class TokenBase: + """Base class for ``Token`` and ``TokenList``. - It represents a single token and has two instance attributes: - ``value`` is the unchanged value of the token and ``ttype`` is - the type of the token. + It has a single instance attribute, ``parent``, which if not ``None`` + represents the ``TokenList`` that contains this token. """ - __slots__ = ('value', 'ttype', 'parent', 'normalized', 'is_keyword', - 'is_group', 'is_whitespace', 'is_newline') + __slots__ = 'parent' - def __init__(self, ttype, value): - value = str(value) - self.value = value - self.ttype = ttype + def __init__(self): self.parent = None - self.is_group = False - self.is_keyword = ttype in T.Keyword - self.is_whitespace = self.ttype in T.Whitespace - self.is_newline = self.ttype in T.Newline - self.normalized = value.upper() if self.is_keyword else value def __str__(self): return self.value @@ -73,19 +63,12 @@ def __repr__(self): return "<{cls} {q}{value}{q} at 0x{id:2X}>".format( id=id(self), **locals()) - def _get_repr_name(self): - return str(self.ttype).split('.')[-1] - def _get_repr_value(self): raw = str(self) if len(raw) > 7: raw = raw[:6] + '...' return re.sub(r'\s+', ' ', raw) - def flatten(self): - """Resolve subgroups.""" - yield self - def match(self, ttype, values, regex=False): """Checks whether the token matches the given arguments. @@ -147,7 +130,43 @@ def has_ancestor(self, other): return False -class TokenList(Token): +class Token(TokenBase): + """"A single token. + + It has some additional instance attributes: + ``value`` is the unchanged value of the token + ``ttype`` is the type of the token + ``normalized`` is the value of the token, converted to uppercase if it + is a keyword + ``is_keyword`` is a boolean indicating if the token is a keyword + ``is_whitespace`` is a boolean indicating if the token is whitespace + ``is_newline`` is a boolean indicating if the token is a newline + character + """ + __slots__ = ('value', 'ttype', 'normalized', 'is_keyword', 'is_whitespace', + 'is_newline') + + is_group = False + + def __init__(self, ttype, value): + super().__init__() + value = str(value) + self.value = value + self.ttype = ttype + self.is_keyword = ttype in T.Keyword + self.is_whitespace = ttype in T.Whitespace + self.is_newline = self.ttype in T.Newline + self.normalized = value.upper() if self.is_keyword else value + + def _get_repr_name(self): + return str(self.ttype).split('.')[-1] + + def flatten(self): + """Resolve subgroups.""" + yield self + + +class TokenList(TokenBase): """A group of tokens. It has an additional instance attribute ``tokens`` which holds a @@ -156,15 +175,25 @@ class TokenList(Token): __slots__ = 'tokens' + is_group = True + ttype = None + is_keyword = False + is_whitespace = False + is_newline = False + def __init__(self, tokens=None): + super().__init__() self.tokens = tokens or [] [setattr(token, 'parent', self) for token in self.tokens] - super().__init__(None, str(self)) - self.is_group = True - def __str__(self): + @property + def value(self): return ''.join(token.value for token in self.flatten()) + @property + def normalized(self): + return self.value + # weird bug # def __len__(self): # return len(self.tokens) @@ -323,7 +352,6 @@ def group_tokens(self, grp_cls, start, end, include_end=True, grp = start grp.tokens.extend(subtokens) del self.tokens[start_idx + 1:end_idx] - grp.value = str(start) else: subtokens = self.tokens[start_idx:end_idx] grp = grp_cls(subtokens)