try:

type = node.type

except AttributeError:

return False

else:

__slots__ = ()

@property

def raw_doc(self):

""" Returns a cleaned version of the docstring token. """

if isinstance(self, Module):

node = self.children[0]

elif isinstance(self, ClassOrFunc):

node = self.children[self.children.index(':') + 1]

if is_node(node, 'suite'): # Normally a suite

node = node.children[2] # -> NEWLINE INDENT stmt

else: # ExprStmt

simple_stmt = self.parent

c = simple_stmt.parent.children

index = c.index(simple_stmt)

if not index:

return ''

node = c[index - 1]

if is_node(node, 'simple_stmt'):

node = node.children[0]

if node.type == 'string':

# TODO We have to check next leaves until there are no new

# leaves anymore that might be part of the docstring. A

# docstring can also look like this: ``'foo' 'bar'

# Returns a literal cleaned version of the ``Token``.

cleaned = cleandoc(literal_eval(node.value))

# Since we want the docstr output to be always unicode, just

# force it.

if is_py3 or isinstance(cleaned, unicode):

return cleaned

else:

return unicode(cleaned, 'UTF-8', 'replace')

"""

__slots__ = ()

def isinstance(self, *cls):

return isinstance(self, cls)

@Python3Method

def get_parent_until(self, classes=(), reverse=False,

include_current=True):

"""

if type(classes) not in (tuple, list):

classes = (classes,)

scope = self if include_current else self.parent

while scope.parent is not None:

# TODO why if classes?

if classes and reverse != scope.isinstance(*classes):

break

scope = scope.parent

return scope

def get_parent_scope(self, include_flows=False):

"""

scope = self.parent

while scope is not None:

if include_flows and isinstance(scope, Flow):

return scope

if scope.is_scope():

break

scope = scope.parent

return scope

def get_definition(self):

if self.type in ('newline', 'dedent', 'indent', 'endmarker'):

raise ValueError('Cannot get the indentation of whitespace or indentation.')

scope = self

while scope.parent is not None:

parent = scope.parent

if scope.isinstance(Node, Leaf) and parent.type != 'simple_stmt':

if scope.type == 'testlist_comp':

try:

if isinstance(scope.children[1], CompFor):

return scope.children[1]

except IndexError:

pass

scope = parent

else:

break

return scope

def assignment_indexes(self):

"""

indexes = []

node = self.parent

compare = self

while node is not None:

if is_node(node, 'testlist_comp', 'testlist_star_expr', 'exprlist'):

for i, child in enumerate(node.children):

if child == compare:

indexes.insert(0, (int(i / 2), node))

break

else:

raise LookupError("Couldn't find the assignment.")

elif isinstance(node, (ExprStmt, CompFor)):

break

compare = node

node = node.parent

return indexes

def is_scope(self):

# Default is not being a scope. Just inherit from Scope.

return False

@abc.abstractmethod

def nodes_to_execute(self, last_added=False):

raise NotImplementedError()

def get_next_sibling(self):

"""

# Can't use index(); we need to test by identity

for i, child in enumerate(self.parent.children):

if child is self:

try:

return self.parent.children[i + 1]

except IndexError:

return None

def get_previous_sibling(self):

"""

# Can't use index(); we need to test by identity

for i, child in enumerate(self.parent.children):

if child is self:

if i == 0:

return None

return self.parent.children[i - 1]

def get_previous_leaf(self):

"""

node = self

while True:

c = node.parent.children

i = c.index(node)

if i == 0:

node = node.parent

if node.parent is None:

raise IndexError('Cannot access the previous element of the first one.')

else:

node = c[i - 1]

break

while True:

try:

node = node.children[-1]

except AttributeError: # A Leaf doesn't have children.

return node

def get_next_leaf(self):

"""

node = self

while True:

c = node.parent.children

i = c.index(node)

if i == len(c) - 1:

node = node.parent

if node.parent is None:

raise IndexError('Cannot access the next element of the last one.')

else:

node = c[i + 1]

break

while True:

try:

node = node.children[0]

except AttributeError: # A Leaf doesn't have children.

__slots__ = ('position_modifier', 'value', 'parent', '_start_pos', 'prefix')

def __init__(self, position_modifier, value, start_pos, prefix=''):

self.position_modifier = position_modifier

self.value = value

self._start_pos = start_pos

self.prefix = prefix

self.parent = None

@property

def start_pos(self):

return self._start_pos[0] + self.position_modifier.line, self._start_pos[1]

@start_pos.setter

def start_pos(self, value):

self._start_pos = value[0] - self.position_modifier.line, value[1]

def get_start_pos_of_prefix(self):

try:

previous_leaf = self

while True:

previous_leaf = previous_leaf.get_previous_leaf()

if previous_leaf.type not in ('indent', 'dedent'):

return previous_leaf.end_pos

except IndexError:

return 1, 0 # It's the first leaf.

@property

def end_pos(self):

return (self._start_pos[0] + self.position_modifier.line,

self._start_pos[1] + len(self.value))

def move(self, line_offset, column_offset):

self._start_pos = (self._start_pos[0] + line_offset,

self._start_pos[1] + column_offset)

def first_leaf(self):

return self

def get_code(self, normalized=False, include_prefix=True):

if normalized:

return self.value

if include_prefix:

return self.prefix + self.value

else:

return self.value

def nodes_to_execute(self, last_added=False):

return []

@utf8_repr

def __repr__(self):

__slots__ = ()

@property

def end_pos(self):

"""

end_pos_line, end_pos_col = self.start_pos

lines = self.value.split('\n')

end_pos_line += len(lines) - 1

# Check for multiline token

if self.start_pos[0] == end_pos_line:

end_pos_col += len(lines[-1])

else:

end_pos_col = len(lines[-1])

return end_pos_line, end_pos_col

@utf8_repr

def __repr__(self):

"""Contains NEWLINE and ENDMARKER tokens."""

__slots__ = ()

type = 'newline'

@utf8_repr

def __repr__(self):

"""

type = 'name'

__slots__ = ()

def __str__(self):

return self.value

def __unicode__(self):

return self.value

def __repr__(self):

return "<%s: %s@%s,%s>" % (type(self).__name__, self.value,

self.start_pos[0], self.start_pos[1])

def is_definition(self):

if self.parent.type in ('power', 'atom_expr'):

# In `self.x = 3` self is not a definition, but x is.

return False

stmt = self.get_definition()

if stmt.type in ('funcdef', 'classdef', 'file_input', 'param'):

return self == stmt.name

elif stmt.type == 'for_stmt':

return self.start_pos < stmt.children[2].start_pos

elif stmt.type == 'try_stmt':

return self.get_previous_sibling() == 'as'

else:

return stmt.type in ('expr_stmt', 'import_name', 'import_from',

'comp_for', 'with_stmt') \

and self in stmt.get_defined_names()

def nodes_to_execute(self, last_added=False):

if last_added is False:

__slots__ = ()

def eval(self):

type = 'operator'

__slots__ = ()

def __str__(self):

return self.value

def __eq__(self, other):

"""

if isinstance(other, Operator):

return self is other

else:

return self.value == other

def __ne__(self, other):

"""Python 2 compatibility."""

return self.value != other

def __hash__(self):

type = 'keyword'

__slots__ = ()

def __eq__(self, other):

"""

if isinstance(other, Keyword):

return self is other

return self.value == other

def __ne__(self, other):

"""Python 2 compatibility."""

return not self.__eq__(other)

def __hash__(self):

"""

__slots__ = ('children', 'parent')

type = None

def __init__(self, children):

"""

for c in children:

c.parent = self

self.children = children

self.parent = None

def move(self, line_offset, column_offset):

"""

for c in self.children:

c.move(line_offset, column_offset)

@property

def start_pos(self):

return self.children[0].start_pos

def get_start_pos_of_prefix(self):

return self.children[0].get_start_pos_of_prefix()

@property

def end_pos(self):

return self.children[-1].end_pos

def get_code(self, normalized=False, include_prefix=True):

# TODO implement normalized (depending on context).

if include_prefix:

return "".join(c.get_code(normalized) for c in self.children)

else:

first = self.children[0].get_code(include_prefix=False)

return first + "".join(c.get_code(normalized) for c in self.children[1:])

@Python3Method

def name_for_position(self, position):

for c in self.children:

if isinstance(c, Leaf):

if isinstance(c, Name) and c.start_pos <= position <= c.end_pos:

return c

else:

result = c.name_for_position(position)

if result is not None:

return result

return None

def get_leaf_for_position(self, position, include_prefixes=False):

for c in self.children:

if include_prefixes:

start_pos = c.get_start_pos_of_prefix()

else:

start_pos = c.start_pos

if start_pos <= position <= c.end_pos:

try:

return c.get_leaf_for_position(position, include_prefixes)

except AttributeError:

while c.type in ('indent', 'dedent'):

# We'd rather not have indents and dedents as a leaf,

# because they don't contain indentation information.

c = c.get_next_leaf()

return c

return None

@Python3Method

def get_statement_for_position(self, pos):

for c in self.children:

if c.start_pos <= pos <= c.end_pos:

if c.type not in ('decorated', 'simple_stmt', 'suite') \

and not isinstance(c, (Flow, ClassOrFunc)):

return c

else:

try:

return c.get_statement_for_position(pos)

except AttributeError:

pass # Must be a non-scope

return None

def first_leaf(self):

try:

return self.children[0].first_leaf()

except AttributeError:

return self.children[0]

def get_next_leaf(self):

"""

c = self.parent.children

index = c.index(self)

if index == len(c) - 1:

# TODO WTF? recursion?

return self.get_next_leaf()

else:

return c[index + 1]

def last_leaf(self):

try:

return self.children[-1].last_leaf()

except AttributeError:

return self.children[-1]

def get_following_comment_same_line(self):

"""

try:

if self.isinstance(ForStmt):

whitespace = self.children[5].first_leaf().prefix

elif self.isinstance(WithStmt):

whitespace = self.children[3].first_leaf().prefix

else:

whitespace = self.last_leaf().get_next_leaf().prefix

except AttributeError:

return None

except ValueError:

# TODO in some particular cases, the tree doesn't seem to be linked

# correctly

return None

if "#" not in whitespace:

return None

comment = whitespace[whitespace.index("#"):]

if "\r" in comment:

comment = comment[:comment.index("\r")]

if "\n" in comment:

comment = comment[:comment.index("\n")]

return comment

@utf8_repr

def __repr__(self):

code = self.get_code().replace('\n', ' ').strip()

if not is_py3:

code = code.encode(encoding, 'replace')

return "<%s: %s@%s,%s>" % \

"""Concrete implementation for interior nodes."""

__slots__ = ('type',)

_IGNORE_EXECUTE_NODES = set([

'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop',

'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators',

'decorated', 'arglist', 'argument', 'exprlist', 'testlist',

'testlist_safe', 'testlist1'

])

def __init__(self, type, children):

"""

super(Node, self).__init__(children)

self.type = type

def nodes_to_execute(self, last_added=False):

"""

result = []

if self.type not in Node._IGNORE_EXECUTE_NODES and not last_added:

result.append(self)

last_added = True

for child in self.children:

result += child.nodes_to_execute(last_added)

return result

def __repr__(self):

"""

__slots__ = ()

type = 'error_node'

def nodes_to_execute(self, last_added=False):

"""

__slots__ = ('original_type')

type = 'error_leaf'

def __init__(self, position_modifier, original_type, value, start_pos, prefix=''):

super(ErrorLeaf, self).__init__(position_modifier, value, start_pos, prefix)

self.original_type = original_type

def __repr__(self):

token_type = token.tok_name[self.original_type]

"""

__slots__ = ('names_dict',)

def __init__(self, children):

super(Scope, self).__init__(children)

@property

def returns(self):

# Needed here for fast_parser, because the fast_parser splits and

# returns will be in "normal" modules.

return self._search_in_scope(ReturnStmt)

@property

def subscopes(self):

return self._search_in_scope(Scope)

@property

def flows(self):

return self._search_in_scope(Flow)

@property

def imports(self):

return self._search_in_scope(Import)

@Python3Method

def _search_in_scope(self, typ):

def scan(children):

elements = []

for element in children:

if isinstance(element, typ):

elements.append(element)

if is_node(element, 'suite', 'simple_stmt', 'decorated') \

or isinstance(element, Flow):

elements += scan(element.children)

return elements

return scan(self.children)

@property

def statements(self):

return self._search_in_scope((ExprStmt, KeywordStatement))

def is_scope(self):

return True

def __repr__(self):

try:

name = self.path

except AttributeError:

try:

name = self.name

except AttributeError:

name = self.command

return "<%s: %s@%s-%s>" % (type(self).__name__, name,

self.start_pos[0], self.end_pos[0])

def walk(self):

yield self

for s in self.subscopes:

for scope in s.walk():

yield scope

for r in self.statements:

while isinstance(r, Flow):

for scope in r.walk():

yield scope

"""

__slots__ = ('path', 'global_names', 'used_names', '_name')

type = 'file_input'

def __init__(self, children):

"""

super(Module, self).__init__(children)

self.path = None # Set later.

@property

@underscore_memoization

def name(self):

""" This is used for the goto functions. """

if self.path is None:

string = '' # no path -> empty name

else:

sep = (re.escape(os.path.sep),) * 2

r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self.path)

# Remove PEP 3149 names

string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))

# Positions are not real, but a module starts at (1, 0)

p = (1, 0)

name = Name(zero_position_modifier, string, p)

name.parent = self

return name

@property

def has_explicit_absolute_import(self):

"""

# TODO this is a strange scan and not fully correct. I think Python's

# parser does it in a different way and scans for the first

# statement/import with a tokenizer (to check for syntax changes like

# the future print statement).

for imp in self.imports:

if imp.type == 'import_from' and imp.level == 0:

for path in imp.paths():

if [str(name) for name in path] == ['__future__', 'absolute_import']:

return True

return False

def nodes_to_execute(self, last_added=False):

# Yield itself, class needs to be executed for decorator checks.

result = []

for child in self.children:

result += child.nodes_to_execute()

type = 'decorator'

__slots__ = ()

def nodes_to_execute(self, last_added=False):

if self.children[-2] == ')':

node = self.children[-3]

if node != '(':

return node.nodes_to_execute()

__slots__ = ()

@property

def name(self):

return self.children[1]

def get_decorators(self):

decorated = self.parent

if is_node(decorated, 'decorated'):

if is_node(decorated.children[0], 'decorators'):

return decorated.children[0].children

else:

return decorated.children[:1]

else:

"""

type = 'classdef'

__slots__ = ()

def __init__(self, children):

super(Class, self).__init__(children)

def get_super_arglist(self):

if self.children[2] != '(': # Has no parentheses

return None

else:

if self.children[3] == ')': # Empty parentheses

return None

else:

return self.children[3]

@property

def doc(self):

"""

docstr = self.raw_doc

for sub in self.subscopes:

if str(sub.name) == '__init__':

return '%s\n\n%s' % (

sub.get_call_signature(func_name=self.name), docstr)

return docstr

def nodes_to_execute(self, last_added=False):

# Yield itself, class needs to be executed for decorator checks.

yield self

# Super arguments.

arglist = self.get_super_arglist()

try:

children = arglist.children

except AttributeError:

if arglist is not None:

for node_to_execute in arglist.nodes_to_execute():

yield node_to_execute

else:

for argument in children:

if argument.type == 'argument':

# metaclass= or list comprehension or */**

raise NotImplementedError('Metaclasses not implemented')

else:

for node_to_execute in argument.nodes_to_execute():

yield node_to_execute

# care for the class suite:

for node in self.children[self.children.index(':'):]:

# This could be easier without the fast parser. But we need to find

# the position of the colon, because everything after it can be a

# part of the class, not just its suite.

for node_to_execute in node.nodes_to_execute():

"""

def check_python2_nested_param(node):

"""

return node.type == 'tfpdef' and node.children[0] == '('

try:

first = argslist_list[0]

except IndexError:

return []

if first.type in ('name', 'tfpdef'):

if check_python2_nested_param(first):

return [first]

else:

return [Param([first], parent)]

elif first == '*':

return [first]

else: # argslist is a `typedargslist` or a `varargslist`.

children = first.children

new_children = []

start = 0

# Start with offset 1, because the end is higher.

for end, child in enumerate(children + [None], 1):

if child is None or child == ',':

param_children = children[start:end]

if param_children: # Could as well be comma and then end.

if check_python2_nested_param(param_children[0]):

new_children += param_children

elif param_children[0] == '*' and param_children[1] == ',':

new_children += param_children

else:

new_children.append(Param(param_children, parent))

start = end