forked from DonJayamanne/pythonVSCode
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy path__init__.py
More file actions
executable file
·379 lines (340 loc) · 16 KB
/
__init__.py
File metadata and controls
executable file
·379 lines (340 loc) · 16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
"""
Evaluation of Python code in |jedi| is based on three assumptions:
* The code uses as least side effects as possible. Jedi understands certain
list/tuple/set modifications, but there's no guarantee that Jedi detects
everything (list.append in different modules for example).
* No magic is being used:
- metaclasses
- ``setattr()`` / ``__import__()``
- writing to ``globals()``, ``locals()``, ``object.__dict__``
* The programmer is not a total dick, e.g. like `this
<https://github.com/davidhalter/jedi/issues/24>`_ :-)
The actual algorithm is based on a principle called lazy evaluation. If you
don't know about it, google it. That said, the typical entry point for static
analysis is calling ``eval_statement``. There's separate logic for
autocompletion in the API, the evaluator is all about evaluating an expression.
Now you need to understand what follows after ``eval_statement``. Let's
make an example::
import datetime
datetime.date.toda# <-- cursor here
First of all, this module doesn't care about completion. It really just cares
about ``datetime.date``. At the end of the procedure ``eval_statement`` will
return the ``date`` class.
To *visualize* this (simplified):
- ``Evaluator.eval_statement`` doesn't do much, because there's no assignment.
- ``Evaluator.eval_element`` cares for resolving the dotted path
- ``Evaluator.find_types`` searches for global definitions of datetime, which
it finds in the definition of an import, by scanning the syntax tree.
- Using the import logic, the datetime module is found.
- Now ``find_types`` is called again by ``eval_element`` to find ``date``
inside the datetime module.
Now what would happen if we wanted ``datetime.date.foo.bar``? Two more
calls to ``find_types``. However the second call would be ignored, because the
first one would return nothing (there's no foo attribute in ``date``).
What if the import would contain another ``ExprStmt`` like this::
from foo import bar
Date = bar.baz
Well... You get it. Just another ``eval_statement`` recursion. It's really
easy. Python can obviously get way more complicated then this. To understand
tuple assignments, list comprehensions and everything else, a lot more code had
to be written.
Jedi has been tested very well, so you can just start modifying code. It's best
to write your own test first for your "new" feature. Don't be scared of
breaking stuff. As long as the tests pass, you're most likely to be fine.
I need to mention now that lazy evaluation is really good because it
only *evaluates* what needs to be *evaluated*. All the statements and modules
that are not used are just being ignored.
"""
import copy
from itertools import chain
from jedi.parser import tree
from jedi import debug
from jedi.evaluate import representation as er
from jedi.evaluate import imports
from jedi.evaluate import recursion
from jedi.evaluate import iterable
from jedi.evaluate.cache import memoize_default
from jedi.evaluate import stdlib
from jedi.evaluate import finder
from jedi.evaluate import compiled
from jedi.evaluate import precedence
from jedi.evaluate import param
from jedi.evaluate import helpers
class Evaluator(object):
def __init__(self, grammar):
self.grammar = grammar
self.memoize_cache = {} # for memoize decorators
# To memorize modules -> equals `sys.modules`.
self.modules = {} # like `sys.modules`.
self.compiled_cache = {} # see `compiled.create()`
self.recursion_detector = recursion.RecursionDetector()
self.execution_recursion_detector = recursion.ExecutionRecursionDetector()
self.analysis = []
def wrap(self, element):
if isinstance(element, tree.Class):
return er.Class(self, element)
elif isinstance(element, tree.Function):
if isinstance(element, tree.Lambda):
return er.LambdaWrapper(self, element)
else:
return er.Function(self, element)
elif isinstance(element, (tree.Module)) \
and not isinstance(element, er.ModuleWrapper):
return er.ModuleWrapper(self, element)
else:
return element
def find_types(self, scope, name_str, position=None, search_global=False,
is_goto=False):
"""
This is the search function. The most important part to debug.
`remove_statements` and `filter_statements` really are the core part of
this completion.
:param position: Position of the last statement -> tuple of line, column
:return: List of Names. Their parents are the types.
"""
f = finder.NameFinder(self, scope, name_str, position)
scopes = f.scopes(search_global)
if is_goto:
return f.filter_name(scopes)
return f.find(scopes, search_global)
@memoize_default(default=[], evaluator_is_first_arg=True)
@recursion.recursion_decorator
@debug.increase_indent
def eval_statement(self, stmt, seek_name=None):
"""
The starting point of the completion. A statement always owns a call
list, which are the calls, that a statement does. In case multiple
names are defined in the statement, `seek_name` returns the result for
this name.
:param stmt: A `tree.ExprStmt`.
"""
debug.dbg('eval_statement %s (%s)', stmt, seek_name)
types = self.eval_element(stmt.get_rhs())
if seek_name:
types = finder.check_tuple_assignments(types, seek_name)
first_operation = stmt.first_operation()
if first_operation not in ('=', None) and not isinstance(stmt, er.InstanceElement): # TODO don't check for this.
# `=` is always the last character in aug assignments -> -1
operator = copy.copy(first_operation)
operator.value = operator.value[:-1]
name = str(stmt.get_defined_names()[0])
parent = self.wrap(stmt.get_parent_scope())
left = self.find_types(parent, name, stmt.start_pos, search_global=True)
if isinstance(stmt.get_parent_until(tree.ForStmt), tree.ForStmt):
# Iterate through result and add the values, that's possible
# only in for loops without clutter, because they are
# predictable.
for r in types:
left = precedence.calculate(self, left, operator, [r])
types = left
else:
types = precedence.calculate(self, left, operator, types)
debug.dbg('eval_statement result %s', types)
return types
@memoize_default(evaluator_is_first_arg=True)
def eval_element(self, element):
if isinstance(element, iterable.AlreadyEvaluated):
return list(element)
elif isinstance(element, iterable.MergedNodes):
return iterable.unite(self.eval_element(e) for e in element)
debug.dbg('eval_element %s@%s', element, element.start_pos)
if isinstance(element, (tree.Name, tree.Literal)) or tree.is_node(element, 'atom'):
return self._eval_atom(element)
elif isinstance(element, tree.Keyword):
# For False/True/None
if element.value in ('False', 'True', 'None'):
return [compiled.builtin.get_by_name(element.value)]
else:
return []
elif element.isinstance(tree.Lambda):
return [er.LambdaWrapper(self, element)]
elif element.isinstance(er.LambdaWrapper):
return [element] # TODO this is no real evaluation.
elif element.type == 'expr_stmt':
return self.eval_statement(element)
elif element.type == 'power':
types = self._eval_atom(element.children[0])
for trailer in element.children[1:]:
if trailer == '**': # has a power operation.
raise NotImplementedError
types = self.eval_trailer(types, trailer)
return types
elif element.type in ('testlist_star_expr', 'testlist',):
# The implicit tuple in statements.
return [iterable.ImplicitTuple(self, element)]
elif element.type in ('not_test', 'factor'):
types = self.eval_element(element.children[-1])
for operator in element.children[:-1]:
types = list(precedence.factor_calculate(self, types, operator))
return types
elif element.type == 'test':
# `x if foo else y` case.
return (self.eval_element(element.children[0]) +
self.eval_element(element.children[-1]))
elif element.type == 'operator':
# Must be an ellipsis, other operators are not evaluated.
return [] # Ignore for now.
elif element.type == 'dotted_name':
types = self._eval_atom(element.children[0])
for next_name in element.children[2::2]:
types = list(chain.from_iterable(self.find_types(typ, next_name)
for typ in types))
return types
else:
return precedence.calculate_children(self, element.children)
def _eval_atom(self, atom):
"""
Basically to process ``atom`` nodes. The parser sometimes doesn't
generate the node (because it has just one child). In that case an atom
might be a name or a literal as well.
"""
if isinstance(atom, tree.Name):
# This is the first global lookup.
stmt = atom.get_definition()
scope = stmt.get_parent_until(tree.IsScope, include_current=True)
if isinstance(stmt, tree.CompFor):
stmt = stmt.get_parent_until((tree.ClassOrFunc, tree.ExprStmt))
if stmt.type != 'expr_stmt':
# We only need to adjust the start_pos for statements, because
# there the name cannot be used.
stmt = atom
return self.find_types(scope, atom, stmt.start_pos, search_global=True)
elif isinstance(atom, tree.Literal):
return [compiled.create(self, atom.eval())]
else:
c = atom.children
# Parentheses without commas are not tuples.
if c[0] == '(' and not len(c) == 2 \
and not(tree.is_node(c[1], 'testlist_comp')
and len(c[1].children) > 1):
return self.eval_element(c[1])
try:
comp_for = c[1].children[1]
except (IndexError, AttributeError):
pass
else:
if isinstance(comp_for, tree.CompFor):
return [iterable.Comprehension.from_atom(self, atom)]
return [iterable.Array(self, atom)]
def eval_trailer(self, types, trailer):
trailer_op, node = trailer.children[:2]
if node == ')': # `arglist` is optional.
node = ()
new_types = []
for typ in types:
debug.dbg('eval_trailer: %s in scope %s', trailer, typ)
if trailer_op == '.':
new_types += self.find_types(typ, node)
elif trailer_op == '(':
new_types += self.execute(typ, node, trailer)
elif trailer_op == '[':
try:
get = typ.get_index_types
except AttributeError:
debug.warning("TypeError: '%s' object is not subscriptable"
% typ)
else:
new_types += get(self, node)
return new_types
def execute_evaluated(self, obj, *args):
"""
Execute a function with already executed arguments.
"""
args = [iterable.AlreadyEvaluated([arg]) for arg in args]
return self.execute(obj, args)
@debug.increase_indent
def execute(self, obj, arguments=(), trailer=None):
if not isinstance(arguments, param.Arguments):
arguments = param.Arguments(self, arguments, trailer)
if obj.isinstance(er.Function):
obj = obj.get_decorated_func()
debug.dbg('execute: %s %s', obj, arguments)
try:
# Some stdlib functions like super(), namedtuple(), etc. have been
# hard-coded in Jedi to support them.
return stdlib.execute(self, obj, arguments)
except stdlib.NotInStdLib:
pass
try:
func = obj.py__call__
except AttributeError:
debug.warning("no execution possible %s", obj)
return []
else:
types = func(self, arguments)
debug.dbg('execute result: %s in %s', types, obj)
return types
def goto_definition(self, name):
def_ = name.get_definition()
if def_.type == 'expr_stmt' and name in def_.get_defined_names():
return self.eval_statement(def_, name)
call = helpers.call_of_name(name)
return self.eval_element(call)
def goto(self, name):
def resolve_implicit_imports(names):
for name in names:
if isinstance(name.parent, helpers.FakeImport):
# Those are implicit imports.
s = imports.ImportWrapper(self, name)
for n in s.follow(is_goto=True):
yield n
else:
yield name
stmt = name.get_definition()
par = name.parent
if par.type == 'argument' and par.children[1] == '=' and par.children[0] == name:
# Named param goto.
trailer = par.parent
if trailer.type == 'arglist':
trailer = trailer.parent
if trailer.type != 'classdef':
if trailer.type == 'decorator':
types = self.eval_element(trailer.children[1])
else:
i = trailer.parent.children.index(trailer)
to_evaluate = trailer.parent.children[:i]
types = self.eval_element(to_evaluate[0])
for trailer in to_evaluate[1:]:
types = self.eval_trailer(types, trailer)
param_names = []
for typ in types:
try:
params = typ.params
except AttributeError:
pass
else:
param_names += [param.name for param in params
if param.name.value == name.value]
return param_names
elif isinstance(par, tree.ExprStmt) and name in par.get_defined_names():
# Only take the parent, because if it's more complicated than just
# a name it's something you can "goto" again.
return [name]
elif isinstance(par, (tree.Param, tree.Function, tree.Class)) and par.name is name:
return [name]
elif isinstance(stmt, tree.Import):
modules = imports.ImportWrapper(self, name).follow(is_goto=True)
return list(resolve_implicit_imports(modules))
elif par.type == 'dotted_name': # Is a decorator.
index = par.children.index(name)
if index > 0:
new_dotted = helpers.deep_ast_copy(par)
new_dotted.children[index - 1:] = []
types = self.eval_element(new_dotted)
return resolve_implicit_imports(iterable.unite(
self.find_types(typ, name, is_goto=True) for typ in types
))
scope = name.get_parent_scope()
if tree.is_node(name.parent, 'trailer'):
call = helpers.call_of_name(name, cut_own_trailer=True)
types = self.eval_element(call)
return resolve_implicit_imports(iterable.unite(
self.find_types(typ, name, is_goto=True) for typ in types
))
else:
if stmt.type != 'expr_stmt':
# We only need to adjust the start_pos for statements, because
# there the name cannot be used.
stmt = name
return self.find_types(scope, name, stmt.start_pos,
search_global=True, is_goto=True)