# encoding:utf-8 """ The :mod:`parser` module concerns itself with parsing Python source. """ from __future__ import absolute_import, division, print_function, unicode_literals from functools import reduce from . import source, diagnostic, lexer, ast # A few notes about our approach to parsing: # # Python uses an LL(1) parser generator. It's a bit weird, because # the usual reason to choose LL(1) is to make a handwritten parser # possible, however Python's grammar is formulated in a way that # is much more easily recognized if you make an FSM rather than # the usual "if accept(token)..." ladder. So in a way it is # the worst of both worlds. # # We don't use a parser generator because we want to have an unified # grammar for all Python versions, and also have grammar coverage # analysis and nice error recovery. To make the grammar compact, # we use combinators to compose it from predefined fragments, # such as "sequence" or "alternation" or "Kleene star". This easily # gives us one token of lookahead in most cases, but e.g. not # in the following one: # # argument: test | test '=' test # # There are two issues with this. First, in an alternation, the first # variant will be tried (and accepted) earlier. Second, if we reverse # them, by the point it is clear ``'='`` will not be accepted, ``test`` # has already been consumed. # # The way we fix this is by reordering rules so that longest match # comes first, and adding backtracking on alternations (as well as # plus and star, since those have a hidden alternation inside). # # While backtracking can in principle make asymptotical complexity # worse, it never makes parsing syntactically correct code supralinear # with Python's LL(1) grammar, and we could not come up with any # pathological incorrect input as well. # Coverage data _all_rules = [] _all_stmts = {} # Generic LL parsing combinators class Unmatched: pass unmatched = Unmatched() def llrule(loc, expected, cases=1): if loc is None: def decorator(rule): rule.expected = expected return rule else: def decorator(inner_rule): if cases == 1: def rule(*args, **kwargs): result = inner_rule(*args, **kwargs) if result is not unmatched: rule.covered[0] = True return result else: rule = inner_rule rule.loc, rule.expected, rule.covered = \ loc, expected, [False] * cases _all_rules.append(rule) return rule return decorator def action(inner_rule, loc=None): """ A decorator returning a function that first runs ``inner_rule`` and then, if its return value is not None, maps that value using ``mapper``. If the value being mapped is a tuple, it is expanded into multiple arguments. Similar to attaching semantic actions to rules in traditional parser generators. """ def decorator(mapper): @llrule(loc, inner_rule.expected) def outer_rule(parser): result = inner_rule(parser) if result is unmatched: return result if isinstance(result, tuple): return mapper(parser, *result) else: return mapper(parser, result) return outer_rule return decorator def Eps(value=None, loc=None): """A rule that accepts no tokens (epsilon) and returns ``value``.""" @llrule(loc, lambda parser: []) def rule(parser): return value return rule def Tok(kind, loc=None): """A rule that accepts a token of kind ``kind`` and returns it, or returns None.""" @llrule(loc, lambda parser: [kind]) def rule(parser): return parser._accept(kind) return rule def Loc(kind, loc=None): """A rule that accepts a token of kind ``kind`` and returns its location, or returns None.""" @llrule(loc, lambda parser: [kind]) def rule(parser): result = parser._accept(kind) if result is unmatched: return result return result.loc return rule def Rule(name, loc=None): """A proxy for a rule called ``name`` which may not be yet defined.""" @llrule(loc, lambda parser: getattr(parser, name).expected(parser)) def rule(parser): return getattr(parser, name)() return rule def Expect(inner_rule, loc=None): """A rule that executes ``inner_rule`` and emits a diagnostic error if it returns None.""" @llrule(loc, inner_rule.expected) def rule(parser): result = inner_rule(parser) if result is unmatched: expected = reduce(list.__add__, [rule.expected(parser) for rule in parser._errrules]) expected = list(sorted(set(expected))) if len(expected) > 1: expected = " or ".join([", ".join(expected[0:-1]), expected[-1]]) elif len(expected) == 1: expected = expected[0] else: expected = "(impossible)" error_tok = parser._tokens[parser._errindex] error = diagnostic.Diagnostic( "fatal", "unexpected {actual}: expected {expected}", {"actual": error_tok.kind, "expected": expected}, error_tok.loc) parser.diagnostic_engine.process(error) return result return rule def Seq(first_rule, *rest_of_rules, **kwargs): """ A rule that accepts a sequence of tokens satisfying ``rules`` and returns a tuple containing their return values, or None if the first rule was not satisfied. """ @llrule(kwargs.get("loc", None), first_rule.expected) def rule(parser): result = first_rule(parser) if result is unmatched: return result results = [result] for rule in rest_of_rules: result = rule(parser) if result is unmatched: return result results.append(result) return tuple(results) return rule def SeqN(n, *inner_rules, **kwargs): """ A rule that accepts a sequence of tokens satisfying ``rules`` and returns the value returned by rule number ``n``, or None if the first rule was not satisfied. """ @action(Seq(*inner_rules), loc=kwargs.get("loc", None)) def rule(parser, *values): return values[n] return rule def Alt(*inner_rules, **kwargs): """ A rule that expects a sequence of tokens satisfying one of ``rules`` in sequence (a rule is satisfied when it returns anything but None) and returns the return value of that rule, or None if no rules were satisfied. """ loc = kwargs.get("loc", None) expected = lambda parser: reduce(list.__add__, map(lambda x: x.expected(parser), inner_rules)) if loc is not None: @llrule(loc, expected, cases=len(inner_rules)) def rule(parser): data = parser._save() for idx, inner_rule in enumerate(inner_rules): result = inner_rule(parser) if result is unmatched: parser._restore(data, rule=inner_rule) else: rule.covered[idx] = True return result return unmatched else: @llrule(loc, expected, cases=len(inner_rules)) def rule(parser): data = parser._save() for inner_rule in inner_rules: result = inner_rule(parser) if result is unmatched: parser._restore(data, rule=inner_rule) else: return result return unmatched return rule def Opt(inner_rule, loc=None): """Shorthand for ``Alt(inner_rule, Eps())``""" return Alt(inner_rule, Eps(), loc=loc) def Star(inner_rule, loc=None): """ A rule that accepts a sequence of tokens satisfying ``inner_rule`` zero or more times, and returns the returned values in a :class:`list`. """ @llrule(loc, lambda parser: []) def rule(parser): results = [] while True: data = parser._save() result = inner_rule(parser) if result is unmatched: parser._restore(data, rule=inner_rule) return results results.append(result) return rule def Plus(inner_rule, loc=None): """ A rule that accepts a sequence of tokens satisfying ``inner_rule`` one or more times, and returns the returned values in a :class:`list`. """ @llrule(loc, inner_rule.expected) def rule(parser): result = inner_rule(parser) if result is unmatched: return result results = [result] while True: data = parser._save() result = inner_rule(parser) if result is unmatched: parser._restore(data, rule=inner_rule) return results results.append(result) return rule class commalist(list): __slots__ = ("trailing_comma",) def List(inner_rule, separator_tok, trailing, leading=True, loc=None): if not trailing: @action(Seq(inner_rule, Star(SeqN(1, Tok(separator_tok), inner_rule))), loc=loc) def outer_rule(parser, first, rest): return [first] + rest return outer_rule else: # A rule like this: stmt (';' stmt)* [';'] # This doesn't yield itself to combinators above, because disambiguating # another iteration of the Kleene star and the trailing separator # requires two lookahead tokens (naively). separator_rule = Tok(separator_tok) @llrule(loc, inner_rule.expected) def rule(parser): results = commalist() if leading: result = inner_rule(parser) if result is unmatched: return result else: results.append(result) while True: result = separator_rule(parser) if result is unmatched: results.trailing_comma = None return results result_1 = inner_rule(parser) if result_1 is unmatched: results.trailing_comma = result return results else: results.append(result_1) return rule # Python AST specific parser combinators def Newline(loc=None): """A rule that accepts token of kind ``newline`` and returns an empty list.""" @llrule(loc, lambda parser: ["newline"]) def rule(parser): result = parser._accept("newline") if result is unmatched: return result return [] return rule def Oper(klass, *kinds, **kwargs): """ A rule that accepts a sequence of tokens of kinds ``kinds`` and returns an instance of ``klass`` with ``loc`` encompassing the entire sequence or None if the first token is not of ``kinds[0]``. """ @action(Seq(*map(Loc, kinds)), loc=kwargs.get("loc", None)) def rule(parser, *tokens): return klass(loc=tokens[0].join(tokens[-1])) return rule def BinOper(expr_rulename, op_rule, node=ast.BinOp, loc=None): @action(Seq(Rule(expr_rulename), Star(Seq(op_rule, Rule(expr_rulename)))), loc=loc) def rule(parser, lhs, trailers): for (op, rhs) in trailers: lhs = node(left=lhs, op=op, right=rhs, loc=lhs.loc.join(rhs.loc)) return lhs return rule def BeginEnd(begin_tok, inner_rule, end_tok, empty=None, loc=None): @action(Seq(Loc(begin_tok), inner_rule, Loc(end_tok)), loc=loc) def rule(parser, begin_loc, node, end_loc): if node is None: node = empty(parser) # Collection nodes don't have loc yet. If a node has loc at this # point, it means it's an expression passed in parentheses. if node.loc is None and type(node) in [ ast.List, ast.ListComp, ast.Dict, ast.DictComp, ast.Set, ast.SetComp, ast.GeneratorExp, ast.Tuple, ast.Repr, ast.Call, ast.Subscript, ast.arguments]: node.begin_loc, node.end_loc, node.loc = \ begin_loc, end_loc, begin_loc.join(end_loc) return node return rule class Parser: # Generic LL parsing methods def __init__(self, lexer, version, diagnostic_engine): self._init_version(version) self.diagnostic_engine = diagnostic_engine self.lexer = lexer self._tokens = [] self._index = -1 self._errindex = -1 self._errrules = [] self._advance() def _save(self): return self._index def _restore(self, data, rule): self._index = data self._token = self._tokens[self._index] if self._index > self._errindex: # We have advanced since last error self._errindex = self._index self._errrules = [rule] elif self._index == self._errindex: # We're at the same place as last error self._errrules.append(rule) else: # We've backtracked far and are now just failing the # whole parse pass def _advance(self): self._index += 1 if self._index == len(self._tokens): self._tokens.append(self.lexer.next(eof_token=True)) self._token = self._tokens[self._index] def _accept(self, expected_kind): if self._token.kind == expected_kind: result = self._token self._advance() return result return unmatched # Python-specific methods def _init_version(self, version): if version in ((2, 6), (2, 7)): if version == (2, 6): self.with_stmt = self.with_stmt__26 self.atom_6 = self.atom_6__26 else: self.with_stmt = self.with_stmt__27 self.atom_6 = self.atom_6__27 self.except_clause_1 = self.except_clause_1__26 self.classdef = self.classdef__26 self.subscript = self.subscript__26 self.raise_stmt = self.raise_stmt__26 self.comp_if = self.comp_if__26 self.atom = self.atom__26 self.funcdef = self.funcdef__26 self.parameters = self.parameters__26 self.varargslist = self.varargslist__26 self.comparison_1 = self.comparison_1__26 self.exprlist_1 = self.exprlist_1__26 self.testlist_comp_1 = self.testlist_comp_1__26 self.expr_stmt_1 = self.expr_stmt_1__26 self.yield_expr = self.yield_expr__26 return elif version in ((3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5)): if version == (3, 0): self.with_stmt = self.with_stmt__26 # lol else: self.with_stmt = self.with_stmt__27 self.except_clause_1 = self.except_clause_1__30 self.classdef = self.classdef__30 self.subscript = self.subscript__30 self.raise_stmt = self.raise_stmt__30 self.comp_if = self.comp_if__30 self.atom = self.atom__30 self.funcdef = self.funcdef__30 self.parameters = self.parameters__30 if version < (3, 2): self.varargslist = self.varargslist__30 self.typedargslist = self.typedargslist__30 self.comparison_1 = self.comparison_1__30 self.star_expr = self.star_expr__30 self.exprlist_1 = self.exprlist_1__30 self.testlist_comp_1 = self.testlist_comp_1__26 self.expr_stmt_1 = self.expr_stmt_1__26 else: self.varargslist = self.varargslist__32 self.typedargslist = self.typedargslist__32 self.comparison_1 = self.comparison_1__32 self.star_expr = self.star_expr__32 self.exprlist_1 = self.exprlist_1__32 self.testlist_comp_1 = self.testlist_comp_1__32 self.expr_stmt_1 = self.expr_stmt_1__32 if version < (3, 3): self.yield_expr = self.yield_expr__26 else: self.yield_expr = self.yield_expr__33 return raise NotImplementedError("pythonparser.parser.Parser cannot parse Python %s" % str(version)) def _arguments(self, args=None, defaults=None, kwonlyargs=None, kw_defaults=None, vararg=None, kwarg=None, star_loc=None, dstar_loc=None, begin_loc=None, end_loc=None, equals_locs=None, kw_equals_locs=None, loc=None): if args is None: args = [] if defaults is None: defaults = [] if kwonlyargs is None: kwonlyargs = [] if kw_defaults is None: kw_defaults = [] if equals_locs is None: equals_locs = [] if kw_equals_locs is None: kw_equals_locs = [] return ast.arguments(args=args, defaults=defaults, kwonlyargs=kwonlyargs, kw_defaults=kw_defaults, vararg=vararg, kwarg=kwarg, star_loc=star_loc, dstar_loc=dstar_loc, begin_loc=begin_loc, end_loc=end_loc, equals_locs=equals_locs, kw_equals_locs=kw_equals_locs, loc=loc) def _arg(self, tok, colon_loc=None, annotation=None): loc = tok.loc if annotation: loc = loc.join(annotation.loc) return ast.arg(arg=tok.value, annotation=annotation, arg_loc=tok.loc, colon_loc=colon_loc, loc=loc) def _empty_arglist(self): return ast.Call(args=[], keywords=[], starargs=None, kwargs=None, star_loc=None, dstar_loc=None, loc=None) def _wrap_tuple(self, elts): assert len(elts) > 0 if len(elts) > 1: return ast.Tuple(ctx=None, elts=elts, loc=elts[0].loc.join(elts[-1].loc), begin_loc=None, end_loc=None) else: return elts[0] def _assignable(self, node, is_delete=False): if isinstance(node, ast.Name) or isinstance(node, ast.Subscript) or \ isinstance(node, ast.Attribute) or isinstance(node, ast.Starred): return node elif (isinstance(node, ast.List) or isinstance(node, ast.Tuple)) and \ any(node.elts): node.elts = [self._assignable(elt, is_delete) for elt in node.elts] return node else: if is_delete: error = diagnostic.Diagnostic( "fatal", "cannot delete this expression", {}, node.loc) else: error = diagnostic.Diagnostic( "fatal", "cannot assign to this expression", {}, node.loc) self.diagnostic_engine.process(error) def add_flags(self, flags): if "print_function" in flags: self.lexer.print_function = True if "unicode_literals" in flags: self.lexer.unicode_literals = True # Grammar @action(Expect(Alt(Newline(), Rule("simple_stmt"), SeqN(0, Rule("compound_stmt"), Newline())))) def single_input(self, body): """single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE""" loc = None if body != []: loc = body[0].loc return ast.Interactive(body=body, loc=loc) @action(Expect(SeqN(0, Star(Alt(Newline(), Rule("stmt"))), Tok("eof")))) def file_input(parser, body): """file_input: (NEWLINE | stmt)* ENDMARKER""" body = reduce(list.__add__, body, []) loc = None if body != []: loc = body[0].loc return ast.Module(body=body, loc=loc) @action(Expect(SeqN(0, Rule("testlist"), Star(Tok("newline")), Tok("eof")))) def eval_input(self, expr): """eval_input: testlist NEWLINE* ENDMARKER""" return ast.Expression(body=[expr], loc=expr.loc) @action(Seq(Loc("@"), List(Tok("ident"), ".", trailing=False), Opt(BeginEnd("(", Opt(Rule("arglist")), ")", empty=_empty_arglist)), Loc("newline"))) def decorator(self, at_loc, idents, call_opt, newline_loc): """decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE""" root = idents[0] dec_loc = root.loc expr = ast.Name(id=root.value, ctx=None, loc=root.loc) for ident in idents[1:]: dot_loc = ident.loc.begin() dot_loc.begin_pos -= 1 dec_loc = dec_loc.join(ident.loc) expr = ast.Attribute(value=expr, attr=ident.value, ctx=None, loc=expr.loc.join(ident.loc), attr_loc=ident.loc, dot_loc=dot_loc) if call_opt: call_opt.func = expr call_opt.loc = dec_loc.join(call_opt.loc) expr = call_opt return at_loc, expr decorators = Plus(Rule("decorator")) """decorators: decorator+""" @action(Seq(Rule("decorators"), Alt(Rule("classdef"), Rule("funcdef")))) def decorated(self, decorators, classfuncdef): """decorated: decorators (classdef | funcdef)""" classfuncdef.at_locs = list(map(lambda x: x[0], decorators)) classfuncdef.decorator_list = list(map(lambda x: x[1], decorators)) classfuncdef.loc = classfuncdef.loc.join(decorators[0][0]) return classfuncdef @action(Seq(Loc("def"), Tok("ident"), Rule("parameters"), Loc(":"), Rule("suite"))) def funcdef__26(self, def_loc, ident_tok, args, colon_loc, suite): """(2.6, 2.7) funcdef: 'def' NAME parameters ':' suite""" return ast.FunctionDef(name=ident_tok.value, args=args, returns=None, body=suite, decorator_list=[], at_locs=[], keyword_loc=def_loc, name_loc=ident_tok.loc, colon_loc=colon_loc, arrow_loc=None, loc=def_loc.join(suite[-1].loc)) @action(Seq(Loc("def"), Tok("ident"), Rule("parameters"), Opt(Seq(Loc("->"), Rule("test"))), Loc(":"), Rule("suite"))) def funcdef__30(self, def_loc, ident_tok, args, returns_opt, colon_loc, suite): """(3.0-) funcdef: 'def' NAME parameters ['->' test] ':' suite""" arrow_loc = returns = None if returns_opt: arrow_loc, returns = returns_opt return ast.FunctionDef(name=ident_tok.value, args=args, returns=returns, body=suite, decorator_list=[], at_locs=[], keyword_loc=def_loc, name_loc=ident_tok.loc, colon_loc=colon_loc, arrow_loc=arrow_loc, loc=def_loc.join(suite[-1].loc)) parameters__26 = BeginEnd("(", Opt(Rule("varargslist")), ")", empty=_arguments) """(2.6, 2.7) parameters: '(' [varargslist] ')'""" parameters__30 = BeginEnd("(", Opt(Rule("typedargslist")), ")", empty=_arguments) """(3.0) parameters: '(' [typedargslist] ')'""" varargslist__26_1 = Seq(Rule("fpdef"), Opt(Seq(Loc("="), Rule("test")))) @action(Seq(Loc("**"), Tok("ident"))) def varargslist__26_2(self, dstar_loc, kwarg_tok): return self._arguments(kwarg=self._arg(kwarg_tok), dstar_loc=dstar_loc, loc=dstar_loc.join(kwarg_tok.loc)) @action(Seq(Loc("*"), Tok("ident"), Opt(Seq(Tok(","), Loc("**"), Tok("ident"))))) def varargslist__26_3(self, star_loc, vararg_tok, kwarg_opt): dstar_loc = kwarg = None loc = star_loc.join(vararg_tok.loc) vararg = self._arg(vararg_tok) if kwarg_opt: _, dstar_loc, kwarg_tok = kwarg_opt kwarg = self._arg(kwarg_tok) loc = star_loc.join(kwarg_tok.loc) return self._arguments(vararg=vararg, kwarg=kwarg, star_loc=star_loc, dstar_loc=dstar_loc, loc=loc) @action(Eps(value=())) def varargslist__26_4(self): return self._arguments() @action(Alt(Seq(Star(SeqN(0, varargslist__26_1, Tok(","))), Alt(varargslist__26_2, varargslist__26_3)), Seq(List(varargslist__26_1, ",", trailing=True), varargslist__26_4))) def varargslist__26(self, fparams, args): """ (2.6, 2.7) varargslist: ((fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [',']) """ for fparam, default_opt in fparams: if default_opt: equals_loc, default = default_opt args.equals_locs.append(equals_loc) args.defaults.append(default) elif len(args.defaults) > 0: error = diagnostic.Diagnostic( "fatal", "non-default argument follows default argument", {}, fparam.loc, [args.args[-1].loc.join(args.defaults[-1].loc)]) self.diagnostic_engine.process(error) args.args.append(fparam) def fparam_loc(fparam, default_opt): if default_opt: equals_loc, default = default_opt return fparam.loc.join(default.loc) else: return fparam.loc if args.loc is None: args.loc = fparam_loc(*fparams[0]).join(fparam_loc(*fparams[-1])) elif len(fparams) > 0: args.loc = args.loc.join(fparam_loc(*fparams[0])) return args @action(Tok("ident")) def fpdef_1(self, ident_tok): return ast.arg(arg=ident_tok.value, annotation=None, arg_loc=ident_tok.loc, colon_loc=None, loc=ident_tok.loc) fpdef = Alt(fpdef_1, BeginEnd("(", Rule("fplist"), ")", empty=lambda self: ast.Tuple(elts=[], ctx=None, loc=None))) """fpdef: NAME | '(' fplist ')'""" def _argslist(fpdef_rule, old_style=False): argslist_1 = Seq(fpdef_rule, Opt(Seq(Loc("="), Rule("test")))) @action(Seq(Loc("**"), Tok("ident"))) def argslist_2(self, dstar_loc, kwarg_tok): return self._arguments(kwarg=self._arg(kwarg_tok), dstar_loc=dstar_loc, loc=dstar_loc.join(kwarg_tok.loc)) @action(Seq(Loc("*"), Tok("ident"), Star(SeqN(1, Tok(","), argslist_1)), Opt(Seq(Tok(","), Loc("**"), Tok("ident"))))) def argslist_3(self, star_loc, vararg_tok, fparams, kwarg_opt): dstar_loc = kwarg = None loc = star_loc.join(vararg_tok.loc) vararg = self._arg(vararg_tok) if kwarg_opt: _, dstar_loc, kwarg_tok = kwarg_opt kwarg = self._arg(kwarg_tok) loc = star_loc.join(kwarg_tok.loc) kwonlyargs, kw_defaults, kw_equals_locs = [], [], [] for fparam, default_opt in fparams: if default_opt: equals_loc, default = default_opt kw_equals_locs.append(equals_loc) kw_defaults.append(default) else: kw_defaults.append(None) kwonlyargs.append(fparam) if any(kw_defaults): loc = loc.join(kw_defaults[-1].loc) elif any(kwonlyargs): loc = loc.join(kwonlyargs[-1].loc) return self._arguments(vararg=vararg, kwarg=kwarg, kwonlyargs=kwonlyargs, kw_defaults=kw_defaults, star_loc=star_loc, dstar_loc=dstar_loc, kw_equals_locs=kw_equals_locs, loc=loc) argslist_4 = Alt(argslist_2, argslist_3) @action(Eps(value=())) def argslist_5(self): return self._arguments() if old_style: argslist = Alt(Seq(Star(SeqN(0, argslist_1, Tok(","))), argslist_4), Seq(List(argslist_1, ",", trailing=True), argslist_5)) else: argslist = Alt(Seq(Eps(value=[]), argslist_4), Seq(List(argslist_1, ",", trailing=False), Alt(SeqN(1, Tok(","), Alt(argslist_4, argslist_5)), argslist_5))) def argslist_action(self, fparams, args): for fparam, default_opt in fparams: if default_opt: equals_loc, default = default_opt args.equals_locs.append(equals_loc) args.defaults.append(default) elif len(args.defaults) > 0: error = diagnostic.Diagnostic( "fatal", "non-default argument follows default argument", {}, fparam.loc, [args.args[-1].loc.join(args.defaults[-1].loc)]) self.diagnostic_engine.process(error) args.args.append(fparam) def fparam_loc(fparam, default_opt): if default_opt: equals_loc, default = default_opt return fparam.loc.join(default.loc) else: return fparam.loc if args.loc is None: args.loc = fparam_loc(*fparams[0]).join(fparam_loc(*fparams[-1])) elif len(fparams) > 0: args.loc = args.loc.join(fparam_loc(*fparams[0])) return args return action(argslist)(argslist_action) typedargslist__30 = _argslist(Rule("tfpdef"), old_style=True) """ (3.0, 3.1) typedargslist: ((tfpdef ['=' test] ',')* ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) """ typedargslist__32 = _argslist(Rule("tfpdef")) """ (3.2-) typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) """ varargslist__30 = _argslist(Rule("vfpdef"), old_style=True) """ (3.0, 3.1) varargslist: ((vfpdef ['=' test] ',')* ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) """ varargslist__32 = _argslist(Rule("vfpdef")) """ (3.2-) varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) """ @action(Seq(Tok("ident"), Opt(Seq(Loc(":"), Rule("test"))))) def tfpdef(self, ident_tok, annotation_opt): """(3.0-) tfpdef: NAME [':' test]""" if annotation_opt: colon_loc, annotation = annotation_opt return self._arg(ident_tok, colon_loc, annotation) return self._arg(ident_tok) vfpdef = fpdef_1 """(3.0-) vfpdef: NAME""" @action(List(Rule("fpdef"), ",", trailing=True)) def fplist(self, elts): """fplist: fpdef (',' fpdef)* [',']""" return ast.Tuple(elts=elts, ctx=None, loc=None) stmt = Alt(Rule("simple_stmt"), Rule("compound_stmt")) """stmt: simple_stmt | compound_stmt""" simple_stmt = SeqN(0, List(Rule("small_stmt"), ";", trailing=True), Tok("newline")) """simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE""" small_stmt = Alt(Rule("expr_stmt"), Rule("print_stmt"), Rule("del_stmt"), Rule("pass_stmt"), Rule("flow_stmt"), Rule("import_stmt"), Rule("global_stmt"), Rule("nonlocal_stmt"), Rule("exec_stmt"), Rule("assert_stmt")) """ (2.6, 2.7) small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt) (3.0-) small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) """ expr_stmt_1__26 = Rule("testlist") expr_stmt_1__32 = Rule("testlist_star_expr") @action(Seq(Rule("augassign"), Alt(Rule("yield_expr"), Rule("testlist")))) def expr_stmt_2(self, augassign, rhs_expr): return ast.AugAssign(op=augassign, value=rhs_expr) @action(Star(Seq(Loc("="), Alt(Rule("yield_expr"), Rule("expr_stmt_1"))))) def expr_stmt_3(self, seq): if len(seq) > 0: return ast.Assign(targets=list(map(lambda x: x[1], seq[:-1])), value=seq[-1][1], op_locs=list(map(lambda x: x[0], seq))) else: return None @action(Seq(Rule("expr_stmt_1"), Alt(expr_stmt_2, expr_stmt_3))) def expr_stmt(self, lhs, rhs): """ (2.6, 2.7, 3.0, 3.1) expr_stmt: testlist (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist))*) (3.2-) expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) """ if isinstance(rhs, ast.AugAssign): if isinstance(lhs, ast.Tuple) or isinstance(lhs, ast.List): error = diagnostic.Diagnostic( "fatal", "illegal expression for augmented assignment", {}, rhs.op.loc, [lhs.loc]) self.diagnostic_engine.process(error) else: rhs.target = self._assignable(lhs) rhs.loc = rhs.target.loc.join(rhs.value.loc) return rhs elif rhs is not None: rhs.targets = list(map(self._assignable, [lhs] + rhs.targets)) rhs.loc = lhs.loc.join(rhs.value.loc) return rhs else: return ast.Expr(value=lhs, loc=lhs.loc) testlist_star_expr = action( List(Alt(Rule("test"), Rule("star_expr")), ",", trailing=True)) \ (_wrap_tuple) """(3.2-) testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']""" augassign = Alt(Oper(ast.Add, "+="), Oper(ast.Sub, "-="), Oper(ast.MatMult, "@="), Oper(ast.Mult, "*="), Oper(ast.Div, "/="), Oper(ast.Mod, "%="), Oper(ast.BitAnd, "&="), Oper(ast.BitOr, "|="), Oper(ast.BitXor, "^="), Oper(ast.LShift, "<<="), Oper(ast.RShift, ">>="), Oper(ast.Pow, "**="), Oper(ast.FloorDiv, "//=")) """augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=')""" @action(List(Rule("test"), ",", trailing=True)) def print_stmt_1(self, values): nl, loc = True, values[-1].loc if values.trailing_comma: nl, loc = False, values.trailing_comma.loc return ast.Print(dest=None, values=values, nl=nl, dest_loc=None, loc=loc) @action(Seq(Loc(">>"), Rule("test"), Tok(","), List(Rule("test"), ",", trailing=True))) def print_stmt_2(self, dest_loc, dest, comma_tok, values): nl, loc = True, values[-1].loc if values.trailing_comma: nl, loc = False, values.trailing_comma.loc return ast.Print(dest=dest, values=values, nl=nl, dest_loc=dest_loc, loc=loc) @action(Eps()) def print_stmt_3(self, eps): return ast.Print(dest=None, values=[], nl=True, dest_loc=None, loc=None) @action(Seq(Loc("print"), Alt(print_stmt_1, print_stmt_2, print_stmt_3))) def print_stmt(self, print_loc, stmt): """ (2.6-2.7) print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] ) """ stmt.keyword_loc = print_loc if stmt.loc is None: stmt.loc = print_loc else: stmt.loc = print_loc.join(stmt.loc) return stmt @action(Seq(Loc("del"), List(Rule("expr"), ",", trailing=True))) def del_stmt(self, stmt_loc, exprs): # Python uses exprlist here, but does *not* obey the usual # tuple-wrapping semantics, so we embed the rule directly. """del_stmt: 'del' exprlist""" return ast.Delete(targets=[self._assignable(expr, is_delete=True) for expr in exprs], loc=stmt_loc.join(exprs[-1].loc), keyword_loc=stmt_loc) @action(Loc("pass")) def pass_stmt(self, stmt_loc): """pass_stmt: 'pass'""" return ast.Pass(loc=stmt_loc, keyword_loc=stmt_loc) flow_stmt = Alt(Rule("break_stmt"), Rule("continue_stmt"), Rule("return_stmt"), Rule("raise_stmt"), Rule("yield_stmt")) """flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt""" @action(Loc("break")) def break_stmt(self, stmt_loc): """break_stmt: 'break'""" return ast.Break(loc=stmt_loc, keyword_loc=stmt_loc) @action(Loc("continue")) def continue_stmt(self, stmt_loc): """continue_stmt: 'continue'""" return ast.Continue(loc=stmt_loc, keyword_loc=stmt_loc) @action(Seq(Loc("return"), Opt(Rule("testlist")))) def return_stmt(self, stmt_loc, values): """return_stmt: 'return' [testlist]""" loc = stmt_loc if values: loc = loc.join(values.loc) return ast.Return(value=values, loc=loc, keyword_loc=stmt_loc) @action(Rule("yield_expr")) def yield_stmt(self, expr): """yield_stmt: yield_expr""" return ast.Expr(value=expr, loc=expr.loc) @action(Seq(Loc("raise"), Opt(Seq(Rule("test"), Opt(Seq(Tok(","), Rule("test"), Opt(SeqN(1, Tok(","), Rule("test"))))))))) def raise_stmt__26(self, raise_loc, type_opt): """(2.6, 2.7) raise_stmt: 'raise' [test [',' test [',' test]]]""" type_ = inst = tback = None loc = raise_loc if type_opt: type_, inst_opt = type_opt loc = loc.join(type_.loc) if inst_opt: _, inst, tback = inst_opt loc = loc.join(inst.loc) if tback: loc = loc.join(tback.loc) return ast.Raise(exc=type_, inst=inst, tback=tback, cause=None, keyword_loc=raise_loc, from_loc=None, loc=loc) @action(Seq(Loc("raise"), Opt(Seq(Rule("test"), Opt(Seq(Loc("from"), Rule("test"))))))) def raise_stmt__30(self, raise_loc, exc_opt): """(3.0-) raise_stmt: 'raise' [test ['from' test]]""" exc = from_loc = cause = None loc = raise_loc if exc_opt: exc, cause_opt = exc_opt loc = loc.join(exc.loc) if cause_opt: from_loc, cause = cause_opt loc = loc.join(cause.loc) return ast.Raise(exc=exc, inst=None, tback=None, cause=cause, keyword_loc=raise_loc, from_loc=from_loc, loc=loc) import_stmt = Alt(Rule("import_name"), Rule("import_from")) """import_stmt: import_name | import_from""" @action(Seq(Loc("import"), Rule("dotted_as_names"))) def import_name(self, import_loc, names): """import_name: 'import' dotted_as_names""" return ast.Import(names=names, keyword_loc=import_loc, loc=import_loc.join(names[-1].loc)) @action(Loc(".")) def import_from_1(self, loc): return 1, loc @action(Loc("...")) def import_from_2(self, loc): return 3, loc @action(Seq(Star(Alt(import_from_1, import_from_2)), Rule("dotted_name"))) def import_from_3(self, dots, dotted_name): dots_loc, dots_count = None, 0 if any(dots): dots_loc = dots[0][1].join(dots[-1][1]) dots_count = sum([count for count, loc in dots]) return (dots_loc, dots_count), dotted_name @action(Plus(Alt(import_from_1, import_from_2))) def import_from_4(self, dots): dots_loc = dots[0][1].join(dots[-1][1]) dots_count = sum([count for count, loc in dots]) return (dots_loc, dots_count), None @action(Loc("*")) def import_from_5(self, star_loc): return (None, 0), \ [ast.alias(name="*", asname=None, name_loc=star_loc, as_loc=None, asname_loc=None, loc=star_loc)], \ None @action(Rule("import_as_names")) def import_from_6(self, names): return (None, 0), names, None @action(Seq(Loc("from"), Alt(import_from_3, import_from_4), Loc("import"), Alt(import_from_5, Seq(Loc("("), Rule("import_as_names"), Loc(")")), import_from_6))) def import_from(self, from_loc, module_name, import_loc, names): """ (2.6, 2.7) import_from: ('from' ('.'* dotted_name | '.'+) 'import' ('*' | '(' import_as_names ')' | import_as_names)) (3.0-) # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 'import' ('*' | '(' import_as_names ')' | import_as_names)) """ (dots_loc, dots_count), dotted_name_opt = module_name module_loc = module = None if dotted_name_opt: module_loc, module = dotted_name_opt lparen_loc, names, rparen_loc = names loc = from_loc.join(names[-1].loc) if rparen_loc: loc = loc.join(rparen_loc) if module == "__future__": self.add_flags([x.name for x in names]) return ast.ImportFrom(names=names, module=module, level=dots_count, keyword_loc=from_loc, dots_loc=dots_loc, module_loc=module_loc, import_loc=import_loc, lparen_loc=lparen_loc, rparen_loc=rparen_loc, loc=loc) @action(Seq(Tok("ident"), Opt(Seq(Loc("as"), Tok("ident"))))) def import_as_name(self, name_tok, as_name_opt): """import_as_name: NAME ['as' NAME]""" asname_name = asname_loc = as_loc = None loc = name_tok.loc if as_name_opt: as_loc, asname = as_name_opt asname_name = asname.value asname_loc = asname.loc loc = loc.join(asname.loc) return ast.alias(name=name_tok.value, asname=asname_name, loc=loc, name_loc=name_tok.loc, as_loc=as_loc, asname_loc=asname_loc) @action(Seq(Rule("dotted_name"), Opt(Seq(Loc("as"), Tok("ident"))))) def dotted_as_name(self, dotted_name, as_name_opt): """dotted_as_name: dotted_name ['as' NAME]""" asname_name = asname_loc = as_loc = None dotted_name_loc, dotted_name_name = dotted_name loc = dotted_name_loc if as_name_opt: as_loc, asname = as_name_opt asname_name = asname.value asname_loc = asname.loc loc = loc.join(asname.loc) return ast.alias(name=dotted_name_name, asname=asname_name, loc=loc, name_loc=dotted_name_loc, as_loc=as_loc, asname_loc=asname_loc) import_as_names = List(Rule("import_as_name"), ",", trailing=True) """import_as_names: import_as_name (',' import_as_name)* [',']""" dotted_as_names = List(Rule("dotted_as_name"), ",", trailing=False) """dotted_as_names: dotted_as_name (',' dotted_as_name)*""" @action(List(Tok("ident"), ".", trailing=False)) def dotted_name(self, idents): """dotted_name: NAME ('.' NAME)*""" return idents[0].loc.join(idents[-1].loc), \ ".".join(list(map(lambda x: x.value, idents))) @action(Seq(Loc("global"), List(Tok("ident"), ",", trailing=False))) def global_stmt(self, global_loc, names): """global_stmt: 'global' NAME (',' NAME)*""" return ast.Global(names=list(map(lambda x: x.value, names)), name_locs=list(map(lambda x: x.loc, names)), keyword_loc=global_loc, loc=global_loc.join(names[-1].loc)) @action(Seq(Loc("exec"), Rule("expr"), Opt(Seq(Loc("in"), Rule("test"), Opt(SeqN(1, Loc(","), Rule("test"))))))) def exec_stmt(self, exec_loc, body, in_opt): """(2.6, 2.7) exec_stmt: 'exec' expr ['in' test [',' test]]""" in_loc, globals, locals = None, None, None loc = exec_loc.join(body.loc) if in_opt: in_loc, globals, locals = in_opt if locals: loc = loc.join(locals.loc) else: loc = loc.join(globals.loc) return ast.Exec(body=body, locals=locals, globals=globals, loc=loc, keyword_loc=exec_loc, in_loc=in_loc) @action(Seq(Loc("nonlocal"), List(Tok("ident"), ",", trailing=False))) def nonlocal_stmt(self, nonlocal_loc, names): """(3.0-) nonlocal_stmt: 'nonlocal' NAME (',' NAME)*""" return ast.Nonlocal(names=list(map(lambda x: x.value, names)), name_locs=list(map(lambda x: x.loc, names)), keyword_loc=nonlocal_loc, loc=nonlocal_loc.join(names[-1].loc)) @action(Seq(Loc("assert"), Rule("test"), Opt(SeqN(1, Tok(","), Rule("test"))))) def assert_stmt(self, assert_loc, test, msg): """assert_stmt: 'assert' test [',' test]""" loc = assert_loc.join(test.loc) if msg: loc = loc.join(msg.loc) return ast.Assert(test=test, msg=msg, loc=loc, keyword_loc=assert_loc) @action(Alt(Rule("if_stmt"), Rule("while_stmt"), Rule("for_stmt"), Rule("try_stmt"), Rule("with_stmt"), Rule("funcdef"), Rule("classdef"), Rule("decorated"))) def compound_stmt(self, stmt): """compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated""" return [stmt] @action(Seq(Loc("if"), Rule("test"), Loc(":"), Rule("suite"), Star(Seq(Loc("elif"), Rule("test"), Loc(":"), Rule("suite"))), Opt(Seq(Loc("else"), Loc(":"), Rule("suite"))))) def if_stmt(self, if_loc, test, if_colon_loc, body, elifs, else_opt): """if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]""" stmt = ast.If(orelse=[], else_loc=None, else_colon_loc=None) if else_opt: stmt.else_loc, stmt.else_colon_loc, stmt.orelse = else_opt for elif_ in reversed(elifs): stmt.keyword_loc, stmt.test, stmt.if_colon_loc, stmt.body = elif_ stmt.loc = stmt.keyword_loc.join(stmt.body[-1].loc) if stmt.orelse: stmt.loc = stmt.loc.join(stmt.orelse[-1].loc) stmt = ast.If(orelse=[stmt], else_loc=None, else_colon_loc=None) stmt.keyword_loc, stmt.test, stmt.if_colon_loc, stmt.body = \ if_loc, test, if_colon_loc, body stmt.loc = stmt.keyword_loc.join(stmt.body[-1].loc) if stmt.orelse: stmt.loc = stmt.loc.join(stmt.orelse[-1].loc) return stmt @action(Seq(Loc("while"), Rule("test"), Loc(":"), Rule("suite"), Opt(Seq(Loc("else"), Loc(":"), Rule("suite"))))) def while_stmt(self, while_loc, test, while_colon_loc, body, else_opt): """while_stmt: 'while' test ':' suite ['else' ':' suite]""" stmt = ast.While(test=test, body=body, orelse=[], keyword_loc=while_loc, while_colon_loc=while_colon_loc, else_loc=None, else_colon_loc=None, loc=while_loc.join(body[-1].loc)) if else_opt: stmt.else_loc, stmt.else_colon_loc, stmt.orelse = else_opt stmt.loc = stmt.loc.join(stmt.orelse[-1].loc) return stmt @action(Seq(Loc("for"), Rule("exprlist"), Loc("in"), Rule("testlist"), Loc(":"), Rule("suite"), Opt(Seq(Loc("else"), Loc(":"), Rule("suite"))))) def for_stmt(self, for_loc, target, in_loc, iter, for_colon_loc, body, else_opt): """for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]""" stmt = ast.For(target=self._assignable(target), iter=iter, body=body, orelse=[], keyword_loc=for_loc, in_loc=in_loc, for_colon_loc=for_colon_loc, else_loc=None, else_colon_loc=None, loc=for_loc.join(body[-1].loc)) if else_opt: stmt.else_loc, stmt.else_colon_loc, stmt.orelse = else_opt stmt.loc = stmt.loc.join(stmt.orelse[-1].loc) return stmt @action(Seq(Plus(Seq(Rule("except_clause"), Loc(":"), Rule("suite"))), Opt(Seq(Loc("else"), Loc(":"), Rule("suite"))), Opt(Seq(Loc("finally"), Loc(":"), Rule("suite"))))) def try_stmt_1(self, clauses, else_opt, finally_opt): handlers = [] for clause in clauses: handler, handler.colon_loc, handler.body = clause handler.loc = handler.loc.join(handler.body[-1].loc) handlers.append(handler) else_loc, else_colon_loc, orelse = None, None, [] loc = handlers[-1].loc if else_opt: else_loc, else_colon_loc, orelse = else_opt loc = orelse[-1].loc finally_loc, finally_colon_loc, finalbody = None, None, [] if finally_opt: finally_loc, finally_colon_loc, finalbody = finally_opt loc = finalbody[-1].loc stmt = ast.Try(body=None, handlers=handlers, orelse=orelse, finalbody=finalbody, else_loc=else_loc, else_colon_loc=else_colon_loc, finally_loc=finally_loc, finally_colon_loc=finally_colon_loc, loc=loc) return stmt @action(Seq(Loc("finally"), Loc(":"), Rule("suite"))) def try_stmt_2(self, finally_loc, finally_colon_loc, finalbody): return ast.Try(body=None, handlers=[], orelse=[], finalbody=finalbody, else_loc=None, else_colon_loc=None, finally_loc=finally_loc, finally_colon_loc=finally_colon_loc, loc=finalbody[-1].loc) @action(Seq(Loc("try"), Loc(":"), Rule("suite"), Alt(try_stmt_1, try_stmt_2))) def try_stmt(self, try_loc, try_colon_loc, body, stmt): """ try_stmt: ('try' ':' suite ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] | 'finally' ':' suite)) """ stmt.keyword_loc, stmt.try_colon_loc, stmt.body = \ try_loc, try_colon_loc, body stmt.loc = stmt.loc.join(try_loc) return stmt @action(Seq(Loc("with"), Rule("test"), Opt(Rule("with_var")), Loc(":"), Rule("suite"))) def with_stmt__26(self, with_loc, context, with_var, colon_loc, body): """(2.6, 3.0) with_stmt: 'with' test [ with_var ] ':' suite""" if with_var: as_loc, optional_vars = with_var item = ast.withitem(context_expr=context, optional_vars=optional_vars, as_loc=as_loc, loc=context.loc.join(optional_vars.loc)) else: item = ast.withitem(context_expr=context, optional_vars=None, as_loc=None, loc=context.loc) return ast.With(items=[item], body=body, keyword_loc=with_loc, colon_loc=colon_loc, loc=with_loc.join(body[-1].loc)) with_var = Seq(Loc("as"), Rule("expr")) """(2.6, 3.0) with_var: 'as' expr""" @action(Seq(Loc("with"), List(Rule("with_item"), ",", trailing=False), Loc(":"), Rule("suite"))) def with_stmt__27(self, with_loc, items, colon_loc, body): """(2.7, 3.1-) with_stmt: 'with' with_item (',' with_item)* ':' suite""" return ast.With(items=items, body=body, keyword_loc=with_loc, colon_loc=colon_loc, loc=with_loc.join(body[-1].loc)) @action(Seq(Rule("test"), Opt(Seq(Loc("as"), Rule("expr"))))) def with_item(self, context, as_opt): """(2.7, 3.1-) with_item: test ['as' expr]""" if as_opt: as_loc, optional_vars = as_opt return ast.withitem(context_expr=context, optional_vars=optional_vars, as_loc=as_loc, loc=context.loc.join(optional_vars.loc)) else: return ast.withitem(context_expr=context, optional_vars=None, as_loc=None, loc=context.loc) @action(Seq(Alt(Loc("as"), Loc(",")), Rule("test"))) def except_clause_1__26(self, as_loc, name): return as_loc, None, name @action(Seq(Loc("as"), Tok("ident"))) def except_clause_1__30(self, as_loc, name): return as_loc, name, None @action(Seq(Loc("except"), Opt(Seq(Rule("test"), Opt(Rule("except_clause_1")))))) def except_clause(self, except_loc, exc_opt): """ (2.6, 2.7) except_clause: 'except' [test [('as' | ',') test]] (3.0-) except_clause: 'except' [test ['as' NAME]] """ type_ = name = as_loc = name_loc = None loc = except_loc if exc_opt: type_, name_opt = exc_opt loc = loc.join(type_.loc) if name_opt: as_loc, name_tok, name_node = name_opt if name_tok: name = name_tok.value name_loc = name_tok.loc else: name = name_node name_loc = name_node.loc loc = loc.join(name_loc) return ast.ExceptHandler(type=type_, name=name, except_loc=except_loc, as_loc=as_loc, name_loc=name_loc, loc=loc) @action(Plus(Rule("stmt"))) def suite_1(self, stmts): return reduce(list.__add__, stmts, []) suite = Alt(Rule("simple_stmt"), SeqN(2, Tok("newline"), Tok("indent"), suite_1, Tok("dedent"))) """suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT""" # 2.x-only backwards compatibility start testlist_safe = action(List(Rule("old_test"), ",", trailing=False))(_wrap_tuple) """(2.6, 2.7) testlist_safe: old_test [(',' old_test)+ [',']]""" old_test = Alt(Rule("or_test"), Rule("old_lambdef")) """(2.6, 2.7) old_test: or_test | old_lambdef""" @action(Seq(Loc("lambda"), Opt(Rule("varargslist")), Loc(":"), Rule("old_test"))) def old_lambdef(self, lambda_loc, args_opt, colon_loc, body): """(2.6, 2.7) old_lambdef: 'lambda' [varargslist] ':' old_test""" if args_opt is None: args_opt = self._arguments() args_opt.loc = colon_loc.begin() return ast.Lambda(args=args_opt, body=body, lambda_loc=lambda_loc, colon_loc=colon_loc, loc=lambda_loc.join(body.loc)) # 2.x-only backwards compatibility end @action(Seq(Rule("or_test"), Opt(Seq(Loc("if"), Rule("or_test"), Loc("else"), Rule("test"))))) def test_1(self, lhs, rhs_opt): if rhs_opt is not None: if_loc, test, else_loc, orelse = rhs_opt return ast.IfExp(test=test, body=lhs, orelse=orelse, if_loc=if_loc, else_loc=else_loc, loc=lhs.loc.join(orelse.loc)) return lhs test = Alt(test_1, Rule("lambdef")) """test: or_test ['if' or_test 'else' test] | lambdef""" test_nocond = Alt(Rule("or_test"), Rule("lambdef_nocond")) """(3.0-) test_nocond: or_test | lambdef_nocond""" def lambdef_action(self, lambda_loc, args_opt, colon_loc, body): if args_opt is None: args_opt = self._arguments() args_opt.loc = colon_loc.begin() return ast.Lambda(args=args_opt, body=body, lambda_loc=lambda_loc, colon_loc=colon_loc, loc=lambda_loc.join(body.loc)) lambdef = action( Seq(Loc("lambda"), Opt(Rule("varargslist")), Loc(":"), Rule("test"))) \ (lambdef_action) """lambdef: 'lambda' [varargslist] ':' test""" lambdef_nocond = action( Seq(Loc("lambda"), Opt(Rule("varargslist")), Loc(":"), Rule("test_nocond"))) \ (lambdef_action) """(3.0-) lambdef_nocond: 'lambda' [varargslist] ':' test_nocond""" @action(Seq(Rule("and_test"), Star(Seq(Loc("or"), Rule("and_test"))))) def or_test(self, lhs, rhs): """or_test: and_test ('or' and_test)*""" if len(rhs) > 0: return ast.BoolOp(op=ast.Or(), values=[lhs] + list(map(lambda x: x[1], rhs)), loc=lhs.loc.join(rhs[-1][1].loc), op_locs=list(map(lambda x: x[0], rhs))) else: return lhs @action(Seq(Rule("not_test"), Star(Seq(Loc("and"), Rule("not_test"))))) def and_test(self, lhs, rhs): """and_test: not_test ('and' not_test)*""" if len(rhs) > 0: return ast.BoolOp(op=ast.And(), values=[lhs] + list(map(lambda x: x[1], rhs)), loc=lhs.loc.join(rhs[-1][1].loc), op_locs=list(map(lambda x: x[0], rhs))) else: return lhs @action(Seq(Oper(ast.Not, "not"), Rule("not_test"))) def not_test_1(self, op, operand): return ast.UnaryOp(op=op, operand=operand, loc=op.loc.join(operand.loc)) not_test = Alt(not_test_1, Rule("comparison")) """not_test: 'not' not_test | comparison""" comparison_1__26 = Seq(Rule("expr"), Star(Seq(Rule("comp_op"), Rule("expr")))) comparison_1__30 = Seq(Rule("star_expr"), Star(Seq(Rule("comp_op"), Rule("star_expr")))) comparison_1__32 = comparison_1__26 @action(Rule("comparison_1")) def comparison(self, lhs, rhs): """ (2.6, 2.7) comparison: expr (comp_op expr)* (3.0, 3.1) comparison: star_expr (comp_op star_expr)* (3.2-) comparison: expr (comp_op expr)* """ if len(rhs) > 0: return ast.Compare(left=lhs, ops=list(map(lambda x: x[0], rhs)), comparators=list(map(lambda x: x[1], rhs)), loc=lhs.loc.join(rhs[-1][1].loc)) else: return lhs @action(Seq(Opt(Loc("*")), Rule("expr"))) def star_expr__30(self, star_opt, expr): """(3.0, 3.1) star_expr: ['*'] expr""" if star_opt: return ast.Starred(value=expr, ctx=None, star_loc=star_opt, loc=expr.loc.join(star_opt)) return expr @action(Seq(Loc("*"), Rule("expr"))) def star_expr__32(self, star_loc, expr): """(3.0-) star_expr: '*' expr""" return ast.Starred(value=expr, ctx=None, star_loc=star_loc, loc=expr.loc.join(star_loc)) comp_op = Alt(Oper(ast.Lt, "<"), Oper(ast.Gt, ">"), Oper(ast.Eq, "=="), Oper(ast.GtE, ">="), Oper(ast.LtE, "<="), Oper(ast.NotEq, "<>"), Oper(ast.NotEq, "!="), Oper(ast.In, "in"), Oper(ast.NotIn, "not", "in"), Oper(ast.IsNot, "is", "not"), Oper(ast.Is, "is")) """ (2.6, 2.7) comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' (3.0-) comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not' """ expr = BinOper("xor_expr", Oper(ast.BitOr, "|")) """expr: xor_expr ('|' xor_expr)*""" xor_expr = BinOper("and_expr", Oper(ast.BitXor, "^")) """xor_expr: and_expr ('^' and_expr)*""" and_expr = BinOper("shift_expr", Oper(ast.BitAnd, "&")) """and_expr: shift_expr ('&' shift_expr)*""" shift_expr = BinOper("arith_expr", Alt(Oper(ast.LShift, "<<"), Oper(ast.RShift, ">>"))) """shift_expr: arith_expr (('<<'|'>>') arith_expr)*""" arith_expr = BinOper("term", Alt(Oper(ast.Add, "+"), Oper(ast.Sub, "-"))) """arith_expr: term (('+'|'-') term)*""" term = BinOper("factor", Alt(Oper(ast.Mult, "*"), Oper(ast.MatMult, "@"), Oper(ast.Div, "/"), Oper(ast.Mod, "%"), Oper(ast.FloorDiv, "//"))) """term: factor (('*'|'/'|'%'|'//') factor)*""" @action(Seq(Alt(Oper(ast.UAdd, "+"), Oper(ast.USub, "-"), Oper(ast.Invert, "~")), Rule("factor"))) def factor_1(self, op, factor): return ast.UnaryOp(op=op, operand=factor, loc=op.loc.join(factor.loc)) factor = Alt(factor_1, Rule("power")) """factor: ('+'|'-'|'~') factor | power""" @action(Seq(Rule("atom"), Star(Rule("trailer")), Opt(Seq(Loc("**"), Rule("factor"))))) def power(self, atom, trailers, factor_opt): """power: atom trailer* ['**' factor]""" for trailer in trailers: if isinstance(trailer, ast.Attribute) or isinstance(trailer, ast.Subscript): trailer.value = atom elif isinstance(trailer, ast.Call): trailer.func = atom trailer.loc = atom.loc.join(trailer.loc) atom = trailer if factor_opt: op_loc, factor = factor_opt return ast.BinOp(left=atom, op=ast.Pow(loc=op_loc), right=factor, loc=atom.loc.join(factor.loc)) return atom @action(Rule("testlist1")) def atom_1(self, expr): return ast.Repr(value=expr, loc=None) @action(Tok("ident")) def atom_2(self, tok): return ast.Name(id=tok.value, loc=tok.loc, ctx=None) @action(Alt(Tok("int"), Tok("float"), Tok("complex"), Tok("long"))) def atom_3(self, tok): return ast.Num(n=tok.value, loc=tok.loc) @action(Seq(Tok("strbegin"), Tok("strdata"), Tok("strend"))) def atom_4(self, begin_tok, data_tok, end_tok): return ast.Str(s=data_tok.value, begin_loc=begin_tok.loc, end_loc=end_tok.loc, loc=begin_tok.loc.join(end_tok.loc)) @action(Plus(atom_4)) def atom_5(self, strings): joint = "" if all(isinstance(x.s, bytes) for x in strings): joint = b"" return ast.Str(s=joint.join([x.s for x in strings]), begin_loc=strings[0].begin_loc, end_loc=strings[-1].end_loc, loc=strings[0].loc.join(strings[-1].loc)) atom_6__26 = Rule("dictmaker") atom_6__27 = Rule("dictorsetmaker") atom__26 = Alt(BeginEnd("(", Opt(Alt(Rule("yield_expr"), Rule("testlist_comp"))), ")", empty=lambda self: ast.Tuple(elts=[], ctx=None, loc=None)), BeginEnd("[", Opt(Rule("listmaker")), "]", empty=lambda self: ast.List(elts=[], ctx=None, loc=None)), BeginEnd("{", Opt(Rule("atom_6")), "}", empty=lambda self: ast.Dict(keys=[], values=[], colon_locs=[], loc=None)), BeginEnd("`", atom_1, "`"), atom_2, atom_3, atom_5) """ (2.6) atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+) (2.7) atom: ('(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dictorsetmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+) """ @action(Loc("...")) def atom_7(self, loc): return ast.Ellipsis(loc=loc) @action(Alt(Tok("None"), Tok("True"), Tok("False"))) def atom_8(self, tok): if tok.kind == "None": value = None elif tok.kind == "True": value = True elif tok.kind == "False": value = False return ast.NameConstant(value=value, loc=tok.loc) atom__30 = Alt(BeginEnd("(", Opt(Alt(Rule("yield_expr"), Rule("testlist_comp"))), ")", empty=lambda self: ast.Tuple(elts=[], ctx=None, loc=None)), BeginEnd("[", Opt(Rule("testlist_comp__list")), "]", empty=lambda self: ast.List(elts=[], ctx=None, loc=None)), BeginEnd("{", Opt(Rule("dictorsetmaker")), "}", empty=lambda self: ast.Dict(keys=[], values=[], colon_locs=[], loc=None)), atom_2, atom_3, atom_5, atom_7, atom_8) """ (3.0-) atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') """ def list_gen_action(self, lhs, rhs): if rhs is None: # (x) return lhs elif isinstance(rhs, ast.Tuple) or isinstance(rhs, ast.List): rhs.elts = [lhs] + rhs.elts return rhs elif isinstance(rhs, ast.ListComp) or isinstance(rhs, ast.GeneratorExp): rhs.elt = lhs return rhs @action(Rule("list_for")) def listmaker_1(self, compose): return ast.ListComp(generators=compose([]), loc=None) @action(List(Rule("test"), ",", trailing=True, leading=False)) def listmaker_2(self, elts): return ast.List(elts=elts, ctx=None, loc=None) listmaker = action( Seq(Rule("test"), Alt(listmaker_1, listmaker_2))) \ (list_gen_action) """listmaker: test ( list_for | (',' test)* [','] )""" testlist_comp_1__26 = Rule("test") testlist_comp_1__32 = Alt(Rule("test"), Rule("star_expr")) @action(Rule("comp_for")) def testlist_comp_2(self, compose): return ast.GeneratorExp(generators=compose([]), loc=None) @action(List(Rule("testlist_comp_1"), ",", trailing=True, leading=False)) def testlist_comp_3(self, elts): if elts == [] and not elts.trailing_comma: return None else: return ast.Tuple(elts=elts, ctx=None, loc=None) testlist_comp = action( Seq(Rule("testlist_comp_1"), Alt(testlist_comp_2, testlist_comp_3))) \ (list_gen_action) """ (2.6) testlist_gexp: test ( gen_for | (',' test)* [','] ) (2.7, 3.0, 3.1) testlist_comp: test ( comp_for | (',' test)* [','] ) (3.2-) testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) """ @action(Rule("comp_for")) def testlist_comp__list_1(self, compose): return ast.ListComp(generators=compose([]), loc=None) @action(List(Rule("testlist_comp_1"), ",", trailing=True, leading=False)) def testlist_comp__list_2(self, elts): return ast.List(elts=elts, ctx=None, loc=None) testlist_comp__list = action( Seq(Rule("testlist_comp_1"), Alt(testlist_comp__list_1, testlist_comp__list_2))) \ (list_gen_action) """Same grammar as testlist_comp, but different semantic action.""" @action(Seq(Loc("."), Tok("ident"))) def trailer_1(self, dot_loc, ident_tok): return ast.Attribute(attr=ident_tok.value, ctx=None, loc=dot_loc.join(ident_tok.loc), attr_loc=ident_tok.loc, dot_loc=dot_loc) trailer = Alt(BeginEnd("(", Opt(Rule("arglist")), ")", empty=_empty_arglist), BeginEnd("[", Rule("subscriptlist"), "]"), trailer_1) """trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME""" @action(List(Rule("subscript"), ",", trailing=True)) def subscriptlist(self, subscripts): """subscriptlist: subscript (',' subscript)* [',']""" if len(subscripts) == 1: return ast.Subscript(slice=subscripts[0], ctx=None, loc=None) elif all([isinstance(x, ast.Index) for x in subscripts]): elts = [x.value for x in subscripts] loc = subscripts[0].loc.join(subscripts[-1].loc) index = ast.Index(value=ast.Tuple(elts=elts, ctx=None, begin_loc=None, end_loc=None, loc=loc), loc=loc) return ast.Subscript(slice=index, ctx=None, loc=None) else: extslice = ast.ExtSlice(dims=subscripts, loc=subscripts[0].loc.join(subscripts[-1].loc)) return ast.Subscript(slice=extslice, ctx=None, loc=None) @action(Seq(Loc("."), Loc("."), Loc("."))) def subscript_1(self, dot_1_loc, dot_2_loc, dot_3_loc): return ast.Ellipsis(loc=dot_1_loc.join(dot_3_loc)) @action(Seq(Opt(Rule("test")), Loc(":"), Opt(Rule("test")), Opt(Rule("sliceop")))) def subscript_2(self, lower_opt, colon_loc, upper_opt, step_opt): loc = colon_loc if lower_opt: loc = loc.join(lower_opt.loc) if upper_opt: loc = loc.join(upper_opt.loc) step_colon_loc = step = None if step_opt: step_colon_loc, step = step_opt loc = loc.join(step_colon_loc) if step: loc = loc.join(step.loc) return ast.Slice(lower=lower_opt, upper=upper_opt, step=step, loc=loc, bound_colon_loc=colon_loc, step_colon_loc=step_colon_loc) @action(Rule("test")) def subscript_3(self, expr): return ast.Index(value=expr, loc=expr.loc) subscript__26 = Alt(subscript_1, subscript_2, subscript_3) """(2.6, 2.7) subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]""" subscript__30 = Alt(subscript_2, subscript_3) """(3.0-) subscript: test | [test] ':' [test] [sliceop]""" sliceop = Seq(Loc(":"), Opt(Rule("test"))) """sliceop: ':' [test]""" exprlist_1__26 = List(Rule("expr"), ",", trailing=True) exprlist_1__30 = List(Rule("star_expr"), ",", trailing=True) exprlist_1__32 = List(Alt(Rule("expr"), Rule("star_expr")), ",", trailing=True) @action(Rule("exprlist_1")) def exprlist(self, exprs): """ (2.6, 2.7) exprlist: expr (',' expr)* [','] (3.0, 3.1) exprlist: star_expr (',' star_expr)* [','] (3.2-) exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] """ return self._wrap_tuple(exprs) @action(List(Rule("test"), ",", trailing=True)) def testlist(self, exprs): """testlist: test (',' test)* [',']""" return self._wrap_tuple(exprs) @action(List(Seq(Rule("test"), Loc(":"), Rule("test")), ",", trailing=True)) def dictmaker(self, elts): """(2.6) dictmaker: test ':' test (',' test ':' test)* [',']""" return ast.Dict(keys=list(map(lambda x: x[0], elts)), values=list(map(lambda x: x[2], elts)), colon_locs=list(map(lambda x: x[1], elts)), loc=None) dictorsetmaker_1 = Seq(Rule("test"), Loc(":"), Rule("test")) @action(Seq(dictorsetmaker_1, Alt(Rule("comp_for"), List(dictorsetmaker_1, ",", leading=False, trailing=True)))) def dictorsetmaker_2(self, first, elts): if isinstance(elts, commalist): elts.insert(0, first) return ast.Dict(keys=list(map(lambda x: x[0], elts)), values=list(map(lambda x: x[2], elts)), colon_locs=list(map(lambda x: x[1], elts)), loc=None) else: return ast.DictComp(key=first[0], value=first[2], generators=elts([]), colon_loc=first[1], begin_loc=None, end_loc=None, loc=None) @action(Seq(Rule("test"), Alt(Rule("comp_for"), List(Rule("test"), ",", leading=False, trailing=True)))) def dictorsetmaker_3(self, first, elts): if isinstance(elts, commalist): elts.insert(0, first) return ast.Set(elts=elts, loc=None) else: return ast.SetComp(elt=first, generators=elts([]), begin_loc=None, end_loc=None, loc=None) dictorsetmaker = Alt(dictorsetmaker_2, dictorsetmaker_3) """ (2.7-) dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | (test (comp_for | (',' test)* [','])) ) """ @action(Seq(Loc("class"), Tok("ident"), Opt(Seq(Loc("("), List(Rule("test"), ",", trailing=True), Loc(")"))), Loc(":"), Rule("suite"))) def classdef__26(self, class_loc, name_tok, bases_opt, colon_loc, body): """(2.6, 2.7) classdef: 'class' NAME ['(' [testlist] ')'] ':' suite""" bases, lparen_loc, rparen_loc = [], None, None if bases_opt: lparen_loc, bases, rparen_loc = bases_opt return ast.ClassDef(name=name_tok.value, bases=bases, keywords=[], starargs=None, kwargs=None, body=body, decorator_list=[], at_locs=[], keyword_loc=class_loc, lparen_loc=lparen_loc, star_loc=None, dstar_loc=None, rparen_loc=rparen_loc, name_loc=name_tok.loc, colon_loc=colon_loc, loc=class_loc.join(body[-1].loc)) @action(Seq(Loc("class"), Tok("ident"), Opt(Seq(Loc("("), Rule("arglist"), Loc(")"))), Loc(":"), Rule("suite"))) def classdef__30(self, class_loc, name_tok, arglist_opt, colon_loc, body): """(3.0) classdef: 'class' NAME ['(' [testlist] ')'] ':' suite""" arglist, lparen_loc, rparen_loc = [], None, None bases, keywords, starargs, kwargs = [], [], None, None star_loc, dstar_loc = None, None if arglist_opt: lparen_loc, arglist, rparen_loc = arglist_opt bases, keywords, starargs, kwargs = \ arglist.args, arglist.keywords, arglist.starargs, arglist.kwargs star_loc, dstar_loc = arglist.star_loc, arglist.dstar_loc return ast.ClassDef(name=name_tok.value, bases=bases, keywords=keywords, starargs=starargs, kwargs=kwargs, body=body, decorator_list=[], at_locs=[], keyword_loc=class_loc, lparen_loc=lparen_loc, star_loc=star_loc, dstar_loc=dstar_loc, rparen_loc=rparen_loc, name_loc=name_tok.loc, colon_loc=colon_loc, loc=class_loc.join(body[-1].loc)) @action(Seq(Loc("*"), Rule("test"), Star(SeqN(1, Tok(","), Rule("argument"))), Opt(Seq(Tok(","), Loc("**"), Rule("test"))))) def arglist_1(self, star_loc, stararg, postargs, kwarg_opt): dstar_loc = kwarg = None if kwarg_opt: _, dstar_loc, kwarg = kwarg_opt for postarg in postargs: if not isinstance(postarg, ast.keyword): error = diagnostic.Diagnostic( "fatal", "only named arguments may follow *expression", {}, postarg.loc, [star_loc.join(stararg.loc)]) self.diagnostic_engine.process(error) return postargs, \ ast.Call(args=[], keywords=[], starargs=stararg, kwargs=kwarg, star_loc=star_loc, dstar_loc=dstar_loc, loc=None) @action(Seq(Loc("**"), Rule("test"))) def arglist_2(self, dstar_loc, kwarg): return [], \ ast.Call(args=[], keywords=[], starargs=None, kwargs=kwarg, star_loc=None, dstar_loc=dstar_loc, loc=None) @action(Seq(Rule("argument"), Alt(SeqN(1, Tok(","), Alt(Rule("arglist_1"), Rule("arglist_2"), Rule("arglist_3"), Eps())), Eps()))) def arglist_3(self, arg, cont): if cont is None: return [arg], self._empty_arglist() else: args, rest = cont return [arg] + args, rest @action(Alt(Rule("arglist_1"), Rule("arglist_2"), Rule("arglist_3"))) def arglist(self, args, call): """arglist: (argument ',')* (argument [','] | '*' test (',' argument)* [',' '**' test] | '**' test)""" for arg in args: if isinstance(arg, ast.keyword): call.keywords.append(arg) elif len(call.keywords) > 0: error = diagnostic.Diagnostic( "fatal", "non-keyword arg after keyword arg", {}, arg.loc, [call.keywords[-1].loc]) self.diagnostic_engine.process(error) else: call.args.append(arg) return call @action(Seq(Loc("="), Rule("test"))) def argument_1(self, equals_loc, rhs): def thunk(lhs): if not isinstance(lhs, ast.Name): error = diagnostic.Diagnostic( "fatal", "keyword must be an identifier", {}, lhs.loc) self.diagnostic_engine.process(error) return ast.keyword(arg=lhs.id, value=rhs, loc=lhs.loc.join(rhs.loc), arg_loc=lhs.loc, equals_loc=equals_loc) return thunk @action(Opt(Rule("comp_for"))) def argument_2(self, compose_opt): def thunk(lhs): if compose_opt: generators = compose_opt([]) return ast.GeneratorExp(elt=lhs, generators=generators, begin_loc=None, end_loc=None, loc=lhs.loc.join(generators[-1].loc)) return lhs return thunk @action(Seq(Rule("test"), Alt(argument_1, argument_2))) def argument(self, lhs, thunk): # This rule is reformulated to avoid exponential backtracking. """ (2.6) argument: test [gen_for] | test '=' test # Really [keyword '='] test (2.7-) argument: test [comp_for] | test '=' test """ return thunk(lhs) list_iter = Alt(Rule("list_for"), Rule("list_if")) """(2.6, 2.7) list_iter: list_for | list_if""" def list_comp_for_action(self, for_loc, target, in_loc, iter, next_opt): def compose(comprehensions): comp = ast.comprehension( target=target, iter=iter, ifs=[], loc=for_loc.join(iter.loc), for_loc=for_loc, in_loc=in_loc, if_locs=[]) comprehensions += [comp] if next_opt: return next_opt(comprehensions) else: return comprehensions return compose def list_comp_if_action(self, if_loc, cond, next_opt): def compose(comprehensions): comprehensions[-1].ifs.append(cond) comprehensions[-1].if_locs.append(if_loc) comprehensions[-1].loc = comprehensions[-1].loc.join(cond.loc) if next_opt: return next_opt(comprehensions) else: return comprehensions return compose list_for = action( Seq(Loc("for"), Rule("exprlist"), Loc("in"), Rule("testlist_safe"), Opt(Rule("list_iter")))) \ (list_comp_for_action) """(2.6, 2.7) list_for: 'for' exprlist 'in' testlist_safe [list_iter]""" list_if = action( Seq(Loc("if"), Rule("old_test"), Opt(Rule("list_iter")))) \ (list_comp_if_action) """(2.6, 2.7) list_if: 'if' old_test [list_iter]""" comp_iter = Alt(Rule("comp_for"), Rule("comp_if")) """ (2.6) gen_iter: gen_for | gen_if (2.7-) comp_iter: comp_for | comp_if """ comp_for = action( Seq(Loc("for"), Rule("exprlist"), Loc("in"), Rule("or_test"), Opt(Rule("comp_iter")))) \ (list_comp_for_action) """ (2.6) gen_for: 'for' exprlist 'in' or_test [gen_iter] (2.7-) comp_for: 'for' exprlist 'in' or_test [comp_iter] """ comp_if__26 = action( Seq(Loc("if"), Rule("old_test"), Opt(Rule("comp_iter")))) \ (list_comp_if_action) """ (2.6) gen_if: 'if' old_test [gen_iter] (2.7) comp_if: 'if' old_test [comp_iter] """ comp_if__30 = action( Seq(Loc("if"), Rule("test_nocond"), Opt(Rule("comp_iter")))) \ (list_comp_if_action) """ (3.0-) comp_if: 'if' test_nocond [comp_iter] """ testlist1 = action(List(Rule("test"), ",", trailing=False))(_wrap_tuple) """testlist1: test (',' test)*""" @action(Seq(Loc("yield"), Opt(Rule("testlist")))) def yield_expr__26(self, yield_loc, exprs): """(2.6, 2.7, 3.0, 3.1, 3.2) yield_expr: 'yield' [testlist]""" if exprs is not None: return ast.Yield(value=exprs, yield_loc=yield_loc, loc=yield_loc.join(exprs.loc)) else: return ast.Yield(value=None, yield_loc=yield_loc, loc=yield_loc) @action(Seq(Loc("yield"), Opt(Rule("yield_arg")))) def yield_expr__33(self, yield_loc, arg): """(3.3-) yield_expr: 'yield' [yield_arg]""" if isinstance(arg, ast.YieldFrom): arg.yield_loc = yield_loc arg.loc = arg.loc.join(arg.yield_loc) return arg elif arg is not None: return ast.Yield(value=arg, yield_loc=yield_loc, loc=yield_loc.join(arg.loc)) else: return ast.Yield(value=None, yield_loc=yield_loc, loc=yield_loc) @action(Seq(Loc("from"), Rule("test"))) def yield_arg_1(self, from_loc, value): return ast.YieldFrom(value=value, from_loc=from_loc, loc=from_loc.join(value.loc)) yield_arg = Alt(yield_arg_1, Rule("testlist")) """(3.3-) yield_arg: 'from' test | testlist"""