try:

type = node.type

except AttributeError:

return False

else:

__slots__ = ()

@property

def raw_doc(self):

""" Returns a cleaned version of the docstring token. """

if isinstance(self, Module):

node = self.children[0]

elif isinstance(self, ClassOrFunc):

node = self.children[self.children.index(':') + 1]

if is_node(node, 'suite'): # Normally a suite

node = node.children[2] # -> NEWLINE INDENT stmt

else: # ExprStmt

simple_stmt = self.parent

c = simple_stmt.parent.children

index = c.index(simple_stmt)

if not index:

return ''

node = c[index - 1]

if is_node(node, 'simple_stmt'):

node = node.children[0]

if node.type == 'string':

# TODO We have to check next leaves until there are no new

# leaves anymore that might be part of the docstring. A

# docstring can also look like this: ``'foo' 'bar'

# Returns a literal cleaned version of the ``Token``.

cleaned = cleandoc(literal_eval(node.value))

# Since we want the docstr output to be always unicode, just

# force it.

if is_py3 or isinstance(cleaned, unicode):

return cleaned

else:

return unicode(cleaned, 'UTF-8', 'replace')

"""

__slots__ = ()

def isinstance(self, *cls):

return isinstance(self, cls)

@Python3Method

def get_parent_until(self, classes=(), reverse=False,

include_current=True):

"""

if type(classes) not in (tuple, list):

classes = (classes,)

scope = self if include_current else self.parent

while scope.parent is not None:

# TODO why if classes?

if classes and reverse != scope.isinstance(*classes):

break

scope = scope.parent

return scope

def get_parent_scope(self, include_flows=False):

"""

scope = self.parent

while scope is not None:

if include_flows and isinstance(scope, Flow):

return scope

if scope.is_scope():

break

scope = scope.parent

return scope

def is_scope(self):

# Default is not being a scope. Just inherit from Scope.

__slots__ = ('position_modifier', 'value', 'parent', '_start_pos', 'prefix')

def __init__(self, position_modifier, value, start_pos, prefix=''):

self.position_modifier = position_modifier

self.value = value

self._start_pos = start_pos

self.prefix = prefix

self.parent = None

@property

def start_pos(self):

return self._start_pos[0] + self.position_modifier.line, self._start_pos[1]

@start_pos.setter

def start_pos(self, value):

self._start_pos = value[0] - self.position_modifier.line, value[1]

@property

def end_pos(self):

return (self._start_pos[0] + self.position_modifier.line,

self._start_pos[1] + len(self.value))

def move(self, line_offset, column_offset):

self._start_pos = (self._start_pos[0] + line_offset,

self._start_pos[1] + column_offset)

def get_previous(self):

"""

node = self

while True:

c = node.parent.children

i = c.index(self)

if i == 0:

node = node.parent

if node.parent is None:

raise IndexError('Cannot access the previous element of the first one.')

else:

node = c[i - 1]

break

while True:

try:

node = node.children[-1]

except AttributeError: # A Leaf doesn't have children.

return node

def get_code(self):

return self.prefix + self.value

def next_sibling(self):

"""

# Can't use index(); we need to test by identity

for i, child in enumerate(self.parent.children):

if child is self:

try:

return self.parent.children[i + 1]

except IndexError:

return None

def prev_sibling(self):

"""

# Can't use index(); we need to test by identity

for i, child in enumerate(self.parent.children):

if child is self:

if i == 0:

return None

return self.parent.children[i - 1]

@utf8_repr

def __repr__(self):

__slots__ = ()

@property

def end_pos(self):

"""

end_pos_line, end_pos_col = self.start_pos

lines = self.value.split('\n')

end_pos_line += len(lines) - 1

# Check for multiline token

if self.start_pos[0] == end_pos_line:

end_pos_col += len(lines[-1])

else:

end_pos_col = len(lines[-1])

return end_pos_line, end_pos_col

@utf8_repr

def __repr__(self):

"""

type = 'name'

__slots__ = ()

def __str__(self):

return self.value

def __unicode__(self):

return self.value

def __repr__(self):

return "<%s: %s@%s,%s>" % (type(self).__name__, self.value,

self.start_pos[0], self.start_pos[1])

def get_definition(self):

scope = self

while scope.parent is not None:

parent = scope.parent

if scope.isinstance(Node, Name) and parent.type != 'simple_stmt':

if scope.type == 'testlist_comp':

try:

if isinstance(scope.children[1], CompFor):

return scope.children[1]

except IndexError:

pass

scope = parent

else:

break

return scope

def is_definition(self):

stmt = self.get_definition()

if stmt.type in ('funcdef', 'classdef', 'file_input', 'param'):

return self == stmt.name

elif stmt.type == 'for_stmt':

return self.start_pos < stmt.children[2].start_pos

elif stmt.type == 'try_stmt':

return self.prev_sibling() == 'as'

else:

return stmt.type in ('expr_stmt', 'import_name', 'import_from',

'comp_for', 'with_stmt') \

and self in stmt.get_defined_names()

def assignment_indexes(self):

"""

indexes = []

node = self.parent

compare = self

while node is not None:

if is_node(node, 'testlist_comp', 'testlist_star_expr', 'exprlist'):

for i, child in enumerate(node.children):

if child == compare:

indexes.insert(0, int(i / 2))

break

else:

raise LookupError("Couldn't find the assignment.")

elif isinstance(node, (ExprStmt, CompFor)):

break

compare = node

node = node.parent

__slots__ = ()

def eval(self):

type = 'operator'

__slots__ = ()

def __str__(self):

return self.value

def __eq__(self, other):

"""

if isinstance(other, Operator):

return self is other

else:

return self.value == other

def __ne__(self, other):

"""Python 2 compatibility."""

return self.value != other

def __hash__(self):

type = 'keyword'

__slots__ = ()

def __eq__(self, other):

"""

if isinstance(other, Keyword):

return self is other

return self.value == other

def __ne__(self, other):

"""Python 2 compatibility."""

return not self.__eq__(other)

def __hash__(self):

"""

__slots__ = ('children', 'parent')

type = None

def __init__(self, children):

"""

for c in children:

c.parent = self

self.children = children

self.parent = None

def move(self, line_offset, column_offset):

"""

for c in self.children:

c.move(line_offset, column_offset)

@property

def start_pos(self):

return self.children[0].start_pos

@property

def end_pos(self):

return self.children[-1].end_pos

def get_code(self):

return "".join(c.get_code() for c in self.children)

@Python3Method

def name_for_position(self, position):

for c in self.children:

if isinstance(c, Leaf):

if isinstance(c, Name) and c.start_pos <= position <= c.end_pos:

return c

else:

result = c.name_for_position(position)

if result is not None:

return result

return None

@Python3Method

def get_statement_for_position(self, pos):

for c in self.children:

if c.start_pos <= pos <= c.end_pos:

if c.type not in ('decorated', 'simple_stmt', 'suite') \

and not isinstance(c, (Flow, ClassOrFunc)):

return c

else:

try:

return c.get_statement_for_position(pos)

except AttributeError:

pass # Must be a non-scope

return None

def first_leaf(self):

try:

return self.children[0].first_leaf()

except AttributeError:

return self.children[0]

@utf8_repr

def __repr__(self):

code = self.get_code().replace('\n', ' ')

if not is_py3:

code = code.encode(encoding, 'replace')

return "<%s: %s@%s,%s>" % \

"""Concrete implementation for interior nodes."""

__slots__ = ('type',)

def __init__(self, type, children):

"""

super(Node, self).__init__(children)

self.type = type

def __repr__(self):

"""

__slots__ = ('names_dict',)

def __init__(self, children):

super(Scope, self).__init__(children)

@property

def returns(self):

# Needed here for fast_parser, because the fast_parser splits and

# returns will be in "normal" modules.

return self._search_in_scope(ReturnStmt)

@property

def subscopes(self):

return self._search_in_scope(Scope)

@property

def flows(self):

return self._search_in_scope(Flow)

@property

def imports(self):

return self._search_in_scope(Import)

@Python3Method

def _search_in_scope(self, typ):

def scan(children):

elements = []

for element in children:

if isinstance(element, typ):

elements.append(element)

if is_node(element, 'suite', 'simple_stmt', 'decorated') \

or isinstance(element, Flow):

elements += scan(element.children)

return elements

return scan(self.children)

@property

def statements(self):

return self._search_in_scope((ExprStmt, KeywordStatement))

def is_scope(self):

return True

def __repr__(self):

try:

name = self.path

except AttributeError:

try:

name = self.name

except AttributeError:

name = self.command

return "<%s: %s@%s-%s>" % (type(self).__name__, name,

self.start_pos[0], self.end_pos[0])

def walk(self):

yield self

for s in self.subscopes:

for scope in s.walk():

yield scope

for r in self.statements:

while isinstance(r, Flow):

for scope in r.walk():

yield scope

"""

__slots__ = ('path', 'global_names', 'used_names', '_name',

'error_statement_stacks')

type = 'file_input'

def __init__(self, children):

"""

super(Module, self).__init__(children)

self.path = None # Set later.

@property

@cache.underscore_memoization

def name(self):

""" This is used for the goto functions. """

if self.path is None:

string = '' # no path -> empty name

else:

sep = (re.escape(os.path.sep),) * 2

r = re.search(r'([^%s]*?)(%s__init__)?(\.py|\.so)?$' % sep, self.path)

# Remove PEP 3149 names

string = re.sub('\.[a-z]+-\d{2}[mud]{0,3}$', '', r.group(1))

# Positions are not real, but a module starts at (1, 0)

p = (1, 0)

name = Name(zero_position_modifier, string, p)

name.parent = self

return name

@property

def has_explicit_absolute_import(self):

"""

# TODO this is a strange scan and not fully correct. I think Python's

# parser does it in a different way and scans for the first

# statement/import with a tokenizer (to check for syntax changes like

# the future print statement).

for imp in self.imports:

if imp.type == 'import_from' and imp.level == 0:

for path in imp.paths():

if [str(name) for name in path] == ['__future__', 'absolute_import']:

return True

__slots__ = ()

@property

def name(self):

return self.children[1]

def get_decorators(self):

decorated = self.parent

if is_node(decorated, 'decorated'):

if is_node(decorated.children[0], 'decorators'):

return decorated.children[0].children

else:

return decorated.children[:1]

else:

"""

type = 'classdef'

__slots__ = ()

def __init__(self, children):

super(Class, self).__init__(children)

def get_super_arglist(self):

if self.children[2] != '(': # Has no parentheses

return None

else:

if self.children[3] == ')': # Empty parentheses

return None

else:

return self.children[3]

@property

def doc(self):

"""

docstr = self.raw_doc

for sub in self.subscopes:

if str(sub.name) == '__init__':

return '%s\n\n%s' % (

sub.get_call_signature(func_name=self.name), docstr)

"""

def check_python2_nested_param(node):

"""

return node.type == 'tfpdef' and node.children[0] == '('

try:

first = argslist_list[0]

except IndexError:

return []

if first.type in ('name', 'tfpdef'):

if check_python2_nested_param(first):

return []

else:

return [Param([first], parent)]

else: # argslist is a `typedargslist` or a `varargslist`.

children = first.children

params = []

start = 0

# Start with offset 1, because the end is higher.

for end, child in enumerate(children + [None], 1):

if child is None or child == ',':

new_children = children[start:end]

if new_children: # Could as well be comma and then end.

if check_python2_nested_param(new_children[0]):

continue

params.append(Param(new_children, parent))

start = end

"""

__slots__ = ('listeners',)

type = 'funcdef'

def __init__(self, children):

super(Function, self).__init__(children)

self.listeners = set() # not used here, but in evaluation.

parameters = self.children[2] # After `def foo`

parameters.children[1:-1] = _create_params(parameters, parameters.children[1:-1])

@property

def params(self):

return self.children[2].children[1:-1]

@property

def name(self):

return self.children[1] # First token after `def`

@property

def yields(self):

# TODO This is incorrect, yields are also possible in a statement.

return self._search_in_scope(YieldExpr)

def is_generator(self):

return bool(self.yields)

def annotation(self):

try:

return self.children[6] # 6th element: def foo(...) -> bar

except IndexError:

return None

def get_call_signature(self, width=72, func_name=None):

"""

func_name = func_name or self.children[1]

code = unicode(func_name) + self.children[2].get_code()

return '\n'.join(textwrap.wrap(code, width))

@property

def doc(self):

""" Return a document string including call signature. """

docstr = self.raw_doc

"""

type = 'lambda'

__slots__ = ()

def __init__(self, children):

# We don't want to call the Function constructor, call its parent.

super(Function, self).__init__(children)

self.listeners = set() # not used here, but in evaluation.

lst = self.children[1:-2] # After `def foo`

self.children[1:-2] = _create_params(self, lst)

@property

def params(self):

return self.children[1:-2]

def is_generator(self):

return False

def yields(self):

return []

def __repr__(self):

type = 'if_stmt'

__slots__ = ()

def check_nodes(self):

"""

for i, c in enumerate(self.children):

if c in ('elif', 'if'):

yield self.children[i + 1]

def node_in_which_check_node(self, node):

for check_node in reversed(list(self.check_nodes())):

if check_node.start_pos < node.start_pos:

return check_node

def node_after_else(self, node):

"""

for c in self.children:

if c == 'else':

if node.start_pos > c.start_pos:

return True

else:

type = 'try_stmt'

__slots__ = ()

def except_clauses(self):

"""

for node in self.children:

if node.type == 'except_clause':

yield node.children[1]

elif node == 'except':

type = 'with_stmt'

__slots__ = ()

def get_defined_names(self):

names = []

for with_item in self.children[1:-2:2]:

# Check with items for 'as' names.

if is_node(with_item, 'with_item'):

names += _defined_names(with_item.children[2])

return names

def node_from_name(self, name):

node = name

while True:

node = node.parent

if is_node(node, 'with_item'):

__slots__ = ()

def path_for_name(self, name):

try:

# The name may be an alias. If it is, just map it back to the name.

name = self.aliases()[name]

except KeyError:

pass

for path in self.paths():

if name in path:

return path[:path.index(name) + 1]

raise ValueError('Name should be defined in the import itself')

def is_nested(self):

return False # By default, sub classes may overwrite this behavior

def is_star_import(self):

type = 'import_from'

__slots__ = ()

def get_defined_names(self):

return [alias or name for name, alias in self._as_name_tuples()]

def aliases(self):

"""Mapping from alias to its corresponding name."""

return dict((alias, name) for name, alias in self._as_name_tuples()

if alias is not None)

def get_from_names(self):

for n in self.children[1:]:

if n not in ('.', '...'):

break

if is_node(n, 'dotted_name'): # from x.y import

return n.children[::2]

elif n == 'import': # from . import

return []

else: # from x import

return [n]

@property

def level(self):

"""The level parameter of ``__import__``."""

level = 0

for n in self.children[1:]:

if n in ('.', '...'):

level += len(n.value)

else:

break

return level

def _as_name_tuples(self):

last = self.children[-1]

if last == ')':

last = self.children[-2]

elif last == '*':

return # No names defined directly.

if is_node(last, 'import_as_names'):

as_names = last.children[::2]

else:

as_names = [last]

for as_name in as_names:

if as_name.type == 'name':

yield as_name, None

else:

yield as_name.children[::2] # yields x, y -> ``x as y``

def star_import_name(self):

"""

return self.paths()[-1][-1]

def paths(self):