-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathparser.py
More file actions
240 lines (191 loc) · 7.55 KB
/
parser.py
File metadata and controls
240 lines (191 loc) · 7.55 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
"""Utilities parsing and analyzing Python code."""
from __future__ import annotations
import ast
import contextlib
import functools
import inspect
import itertools
import operator
import re
import tokenize
from inspect import Signature
from token import DEDENT, INDENT, NAME, NEWLINE, NUMBER, OP, STRING
from tokenize import COMMENT, NL
from typing import Any
from sphinx.pycode.ast import unparse as ast_unparse
comment_re = re.compile('^\\s*#: ?(.*)\r?\n?$')
indent_re = re.compile('^\\s*$')
emptyline_re = re.compile('^\\s*(#.*)?$')
def get_assign_targets(node: ast.AST) -> list[ast.expr]:
"""Get list of targets from Assign and AnnAssign node."""
pass
def get_lvar_names(node: ast.AST, self: ast.arg | None=None) -> list[str]:
"""Convert assignment-AST to variable names.
This raises `TypeError` if the assignment does not create new variable::
ary[0] = 'foo'
dic["bar"] = 'baz'
# => TypeError
"""
pass
def dedent_docstring(s: str) -> str:
"""Remove common leading indentation from docstring."""
pass
class Token:
"""Better token wrapper for tokenize module."""
def __init__(self, kind: int, value: Any, start: tuple[int, int], end: tuple[int, int], source: str) -> None:
self.kind = kind
self.value = value
self.start = start
self.end = end
self.source = source
def __eq__(self, other: Any) -> bool:
if isinstance(other, int):
return self.kind == other
elif isinstance(other, str):
return self.value == other
elif isinstance(other, list | tuple):
return [self.kind, self.value] == list(other)
elif other is None:
return False
else:
raise ValueError('Unknown value: %r' % other)
def __repr__(self) -> str:
return f'<Token kind={tokenize.tok_name[self.kind]!r} value={self.value.strip()!r}>'
class TokenProcessor:
def __init__(self, buffers: list[str]) -> None:
lines = iter(buffers)
self.buffers = buffers
self.tokens = tokenize.generate_tokens(lambda: next(lines))
self.current: Token | None = None
self.previous: Token | None = None
def get_line(self, lineno: int) -> str:
"""Returns specified line."""
pass
def fetch_token(self) -> Token | None:
"""Fetch the next token from source code.
Returns ``None`` if sequence finished.
"""
pass
def fetch_until(self, condition: Any) -> list[Token]:
"""Fetch tokens until specified token appeared.
.. note:: This also handles parenthesis well.
"""
pass
class AfterCommentParser(TokenProcessor):
"""Python source code parser to pick up comments after assignments.
This parser takes code which starts with an assignment statement,
and returns the comment for the variable if one exists.
"""
def __init__(self, lines: list[str]) -> None:
super().__init__(lines)
self.comment: str | None = None
def fetch_rvalue(self) -> list[Token]:
"""Fetch right-hand value of assignment."""
pass
def parse(self) -> None:
"""Parse the code and obtain comment after assignment."""
pass
class VariableCommentPicker(ast.NodeVisitor):
"""Python source code parser to pick up variable comments."""
def __init__(self, buffers: list[str], encoding: str) -> None:
self.counter = itertools.count()
self.buffers = buffers
self.encoding = encoding
self.context: list[str] = []
self.current_classes: list[str] = []
self.current_function: ast.FunctionDef | None = None
self.comments: dict[tuple[str, str], str] = {}
self.annotations: dict[tuple[str, str], str] = {}
self.previous: ast.AST | None = None
self.deforders: dict[str, int] = {}
self.finals: list[str] = []
self.overloads: dict[str, list[Signature]] = {}
self.typing: str | None = None
self.typing_final: str | None = None
self.typing_overload: str | None = None
super().__init__()
def get_qualname_for(self, name: str) -> list[str] | None:
"""Get qualified name for given object as a list of string(s)."""
pass
def get_self(self) -> ast.arg | None:
"""Returns the name of the first argument if in a function."""
pass
def get_line(self, lineno: int) -> str:
"""Returns specified line."""
pass
def visit(self, node: ast.AST) -> None:
"""Updates self.previous to the given node."""
pass
def visit_Import(self, node: ast.Import) -> None:
"""Handles Import node and record the order of definitions."""
pass
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
"""Handles Import node and record the order of definitions."""
pass
def visit_Assign(self, node: ast.Assign) -> None:
"""Handles Assign node and pick up a variable comment."""
pass
def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
"""Handles AnnAssign node and pick up a variable comment."""
pass
def visit_Expr(self, node: ast.Expr) -> None:
"""Handles Expr node and pick up a comment if string."""
pass
def visit_Try(self, node: ast.Try) -> None:
"""Handles Try node and processes body and else-clause.
.. note:: pycode parser ignores objects definition in except-clause.
"""
pass
def visit_ClassDef(self, node: ast.ClassDef) -> None:
"""Handles ClassDef node and set context."""
pass
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
"""Handles FunctionDef node and set context."""
pass
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
"""Handles AsyncFunctionDef node and set context."""
pass
class DefinitionFinder(TokenProcessor):
"""Python source code parser to detect location of functions,
classes and methods.
"""
def __init__(self, lines: list[str]) -> None:
super().__init__(lines)
self.decorator: Token | None = None
self.context: list[str] = []
self.indents: list[tuple[str, str | None, int | None]] = []
self.definitions: dict[str, tuple[str, int, int]] = {}
def add_definition(self, name: str, entry: tuple[str, int, int]) -> None:
"""Add a location of definition."""
pass
def parse(self) -> None:
"""Parse the code to obtain location of definitions."""
pass
def parse_definition(self, typ: str) -> None:
"""Parse AST of definition."""
pass
def finalize_block(self) -> None:
"""Finalize definition block."""
pass
class Parser:
"""Python source code parser to pick up variable comments.
This is a better wrapper for ``VariableCommentPicker``.
"""
def __init__(self, code: str, encoding: str='utf-8') -> None:
self.code = filter_whitespace(code)
self.encoding = encoding
self.annotations: dict[tuple[str, str], str] = {}
self.comments: dict[tuple[str, str], str] = {}
self.deforders: dict[str, int] = {}
self.definitions: dict[str, tuple[str, int, int]] = {}
self.finals: list[str] = []
self.overloads: dict[str, list[Signature]] = {}
def parse(self) -> None:
"""Parse the source code."""
pass
def parse_comments(self) -> None:
"""Parse the code and pick up comments."""
pass
def parse_definition(self) -> None:
"""Parse the location of definitions from the code."""
pass