Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Fix/misc annotations #112

Merged
juanjux merged 6 commits into from
Oct 2, 2017
Merged

Fix/misc annotations #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
48cafaf
Add Module to Modules. Remove Function, Declaration and Call from Arg…
Sep 28, 2017
308049c
Compare node and children fixes
Sep 28, 2017
5dee52a
Misc annotation fixes
Sep 28, 2017
f122947
Set Function+(Declaration|Call) for arguments
Sep 29, 2017
27fee73
Remove Identifier from the arguments grouping node. Remove dups
Sep 29, 2017
f09a524
Move new tests to the new location
Oct 2, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
363 changes: 363 additions & 0 deletions ANNOTATION.md
View file
Open in desktop

Large diffs are not rendered by default.

52 changes: 24 additions & 28 deletions driver/normalizer/annotation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ var Transformers = []transformer.Tranformer{
// https://godoc.org/gopkg.in/bblfsh/sdk.v1/uast/ann
var AnnotationRules = On(Any).Self(
On(Not(pyast.Module)).Error(errors.New("root must be uast.Module")),
On(pyast.Module).Roles(uast.File).Descendants(
On(pyast.Module).Roles(uast.File, uast.Module).Descendants(

// Binary Expressions
On(pyast.BinOp).Roles(uast.Expression, uast.Binary).Children(
Expand Down Expand Up @@ -127,16 +127,17 @@ var AnnotationRules = On(Any).Self(

// FIXME: the FunctionDeclarationReceiver is not set for methods; it should be taken from the parent
// Type node Token (2 levels up) but the SDK doesn't allow this
On(pyast.FunctionDef).Roles(uast.Function, uast.Declaration, uast.Name, uast.Identifier),
On(pyast.FunctionDef).Roles(uast.Function, uast.Declaration, uast.Name, uast.Identifier).Children(
On(pyast.Arguments).Roles(uast.Function, uast.Declaration, uast.Incomplete, uast.Argument).Children(
On(HasInternalRole("args")).Roles(uast.Function, uast.Declaration, uast.Argument, uast.Name, uast.Identifier),
On(HasInternalRole("vararg")).Roles(uast.Function, uast.Declaration, uast.Argument, uast.ArgsList, uast.Name, uast.Identifier),
On(HasInternalRole("kwarg")).Roles(uast.Function, uast.Declaration, uast.Argument, uast.ArgsList, uast.Map, uast.Name, uast.Identifier),
On(HasInternalRole("kwonlyargs")).Roles(uast.Function, uast.Declaration, uast.Argument, uast.ArgsList, uast.Map, uast.Name, uast.Identifier),
),
),
On(pyast.AsyncFunctionDef).Roles(uast.Function, uast.Declaration, uast.Name, uast.Identifier, uast.Incomplete),
On(pyast.FuncDecorators).Roles(uast.Function, uast.Declaration, uast.Call, uast.Incomplete),
On(pyast.FuncDefBody).Roles(uast.Function, uast.Declaration, uast.Body),
// FIXME: arguments is a Groping node, update it we get a "Grouper" or "Container" role
On(HasInternalRole("arguments")).Roles(uast.Function, uast.Declaration, uast.Argument, uast.Incomplete),
On(HasInternalRole("args")).Roles(uast.Function, uast.Declaration, uast.Argument, uast.Name, uast.Identifier),
On(HasInternalRole("vararg")).Roles(uast.Function, uast.Declaration, uast.Argument, uast.ArgsList, uast.Name, uast.Identifier),
On(HasInternalRole("kwarg")).Roles(uast.Function, uast.Declaration, uast.Argument, uast.ArgsList, uast.Map, uast.Name, uast.Identifier),
On(HasInternalRole("kwonlyargs")).Roles(uast.Function, uast.Declaration, uast.Argument, uast.ArgsList, uast.Map, uast.Name, uast.Identifier),
// Default arguments: Python's AST puts default arguments on a sibling list to the one of
// arguments that must be mapped to the arguments right-aligned like:
// a, b=2, c=3 ->
Expand All @@ -155,9 +156,9 @@ var AnnotationRules = On(Any).Self(
On(pyast.Name).Roles(uast.Identifier, uast.Qualified)),

On(pyast.Call).Roles(uast.Function, uast.Call, uast.Expression).Children(
On(HasInternalRole("args")).Roles(uast.Call, uast.Argument, uast.Positional),
On(HasInternalRole("keywords")).Roles(uast.Call, uast.Argument, uast.Name).Children(
On(HasInternalRole("value")).Roles(uast.Call, uast.Argument, uast.Value),
On(HasInternalRole("args")).Roles(uast.Function, uast.Call, uast.Positional, uast.Argument, uast.Name),
On(HasInternalRole("keywords")).Roles(uast.Function, uast.Call, uast.Argument, uast.Name).Children(
On(HasInternalRole("value")).Roles(uast.Argument, uast.Value),
),
On(HasInternalRole("func")).Self(
On(pyast.Name).Roles(uast.Call, uast.Callee),
Expand Down Expand Up @@ -217,21 +218,19 @@ var AnnotationRules = On(Any).Self(
On(pyast.Break).Roles(uast.Break, uast.Statement),
On(pyast.Continue).Roles(uast.Continue, uast.Statement),

// - Compare.ops (internaluast.Type): [uast.LessThan, uast.LessThan]
// - Compare.comparators (internaluast.Type): ['a', 10]
// The current mapping is:
// - left: uast.Expression, uast.Binary, uast.Left
// - Compare.ops: uast.Expression, uast.Binary, uast.Operator
// - Compare.comparators: uast.Expression, uast.Binary, uast.Right
// But this is obviously not correct. To fix this properly we would need
// and SDK feature to mix lists (also needed for default and keyword arguments and
// boolean operators).
// "uast.If that sounds awkward is because it is" (their words)
// Comparison nodes in Python are oddly structured. Probably one if the first
// things that could be changed once we can normalize tree structures. Check:
// https://greentreesnakes.readthedocs.io/en/latest/nodes.html#Compare

// Parent of all comparisons
On(pyast.Compare).Roles(uast.Expression, uast.Binary).Children(
On(pyast.CompareOps).Roles(uast.Expression, uast.Binary, uast.Operator),
On(HasInternalRole("left")).Roles(uast.Expression, uast.Binary, uast.Left),
// Operators
On(pyast.CompareOps).Roles(uast.Expression),
// Leftmost element (the others are the comparators below)
On(HasInternalRole("left")).Roles(uast.Expression, uast.Left),
// These hold the members of the comparison (not the operators)
On(pyast.CompareComparators).Roles(uast.Expression, uast.Right),
),
On(pyast.CompareComparators).Roles(uast.Expression, uast.Binary, uast.Right),
On(pyast.If).Roles(uast.If, uast.Statement).Children(
On(pyast.IfBody).Roles(uast.If, uast.Body, uast.Then),
On(HasInternalRole("test")).Roles(uast.If, uast.Condition),
Expand Down Expand Up @@ -320,10 +319,7 @@ var AnnotationRules = On(Any).Self(
On(HasInternalRole("iter")).Roles(uast.For, uast.Update, uast.Statement),
On(HasInternalRole("target")).Roles(uast.For, uast.Expression),
// FIXME: see the comment on uast.If, uast.Condition above
On(pyast.Compare).Roles(uast.If, uast.Condition, uast.Expression, uast.Binary).Children(
On(pyast.CompareOps).Roles(uast.Expression, uast.Binary, uast.Operator),
On(HasInternalRole("left")).Roles(uast.Expression, uast.Binary, uast.Left),
),
On(pyast.Compare).Roles(uast.If, uast.Condition),
),

On(pyast.Delete).Roles(uast.Statement, uast.Incomplete),
Expand Down
4 changes: 2 additions & 2 deletions fixtures/annotations.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: AnnAssign {
. . . Roles: Operator,Binary,Assignment,Comment,Incomplete
Expand Down Expand Up @@ -140,7 +140,7 @@ Module {
. . . }
. . . Children: {
. . . . 0: arguments {
. . . . . Roles: Function,Declaration,Argument,Name,Identifier
. . . . . Roles: Function,Declaration,Incomplete,Argument
. . . . . Properties: {
. . . . . . internalRole: args
. . . . . . kwarg: <nil>
Expand Down
2 changes: 1 addition & 1 deletion fixtures/aritmeticops.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: Expr {
. . . Roles: Expression
Expand Down
2 changes: 1 addition & 1 deletion fixtures/assert_constant.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: Assert {
. . . Roles: Assert,Statement
Expand Down
2 changes: 1 addition & 1 deletion fixtures/augassign.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: AugAssign {
. . . Roles: Operator,Binary,Assignment,Statement
Expand Down
2 changes: 1 addition & 1 deletion fixtures/bitwise.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: Expr {
. . . Roles: Expression
Expand Down
30 changes: 30 additions & 0 deletions fixtures/classdef.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,13 @@ Module {
. . . . . . . }
. . . . . . . Children: {
. . . . . . . . 0: arguments {
<<<<<<< HEAD
. . . . . . . . . Roles: Function,Declaration,Argument,Name,Identifier
||||||| merged common ancestors
. . . . . . . . . Roles: Function,Declaration,Incomplete,Argument,Name,Identifier
=======
. . . . . . . . . Roles: Function,Declaration,Incomplete,Argument
>>>>>>> Remove Identifier from the arguments grouping node. Remove dups
. . . . . . . . . Properties: {
. . . . . . . . . . internalRole: args
. . . . . . . . . . kwarg: <nil>
Expand Down Expand Up @@ -210,7 +216,13 @@ Module {
. . . . . . . }
. . . . . . . Children: {
. . . . . . . . 0: arguments {
<<<<<<< HEAD
. . . . . . . . . Roles: Function,Declaration,Argument,Name,Identifier
||||||| merged common ancestors
. . . . . . . . . Roles: Function,Declaration,Incomplete,Argument,Name,Identifier
=======
. . . . . . . . . Roles: Function,Declaration,Incomplete,Argument
>>>>>>> Remove Identifier from the arguments grouping node. Remove dups
. . . . . . . . . Properties: {
. . . . . . . . . . internalRole: args
. . . . . . . . . . kwarg: <nil>
Expand Down Expand Up @@ -445,7 +457,13 @@ Module {
. . . . . . . . . }
. . . . . . . . }
. . . . . . . . 2: arguments {
<<<<<<< HEAD
. . . . . . . . . Roles: Function,Declaration,Argument,Name,Identifier
||||||| merged common ancestors
. . . . . . . . . Roles: Function,Declaration,Incomplete,Argument,Name,Identifier
=======
. . . . . . . . . Roles: Function,Declaration,Incomplete,Argument
>>>>>>> Remove Identifier from the arguments grouping node. Remove dups
. . . . . . . . . Properties: {
. . . . . . . . . . internalRole: args
. . . . . . . . . . kwarg: <nil>
Expand All @@ -467,7 +485,13 @@ Module {
. . . . . . . }
. . . . . . . Children: {
. . . . . . . . 0: arguments {
<<<<<<< HEAD
. . . . . . . . . Roles: Function,Declaration,Argument,Name,Identifier
||||||| merged common ancestors
. . . . . . . . . Roles: Function,Declaration,Incomplete,Argument,Name,Identifier
=======
. . . . . . . . . Roles: Function,Declaration,Incomplete,Argument
>>>>>>> Remove Identifier from the arguments grouping node. Remove dups
. . . . . . . . . Properties: {
. . . . . . . . . . internalRole: args
. . . . . . . . . . kwarg: <nil>
Expand Down Expand Up @@ -1006,7 +1030,13 @@ Module {
. . . . . }
. . . . . Children: {
. . . . . . 0: Num {
<<<<<<< HEAD
. . . . . . . Roles: Literal,Number,Expression,Primitive,Function,Declaration,Argument,Name,Identifier,Call,Argument,Positional
||||||| merged common ancestors
. . . . . . . Roles: Literal,Number,Expression,Argument,Name,Identifier,Function,Call,Positional
=======
. . . . . . . Roles: Literal,Number,Expression,Function,Call,Positional,Argument,Name
>>>>>>> Remove Identifier from the arguments grouping node. Remove dups
. . . . . . . TOKEN "5"
. . . . . . . StartPosition: {
. . . . . . . . Offset: 269
Expand Down
2 changes: 1 addition & 1 deletion fixtures/classdef_decorated.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: ClassDef {
. . . Roles: Type,Declaration,Identifier,Statement
Expand Down
2 changes: 1 addition & 1 deletion fixtures/classdef_inheritance.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: ClassDef {
. . . Roles: Type,Declaration,Identifier,Statement
Expand Down
2 changes: 1 addition & 1 deletion fixtures/classdef_metaclass_py2.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: ClassDef {
. . . Roles: Type,Declaration,Identifier,Statement
Expand Down
2 changes: 1 addition & 1 deletion fixtures/classdef_metaclass_py3.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: ClassDef {
. . . Roles: Type,Declaration,Identifier,Statement
Expand Down
2 changes: 1 addition & 1 deletion fixtures/comments.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Status: ok
Errors:
UAST:
Module {
. Roles: File
. Roles: File,Module
. Children: {
. . 0: Assign {
. . . Roles: Binary,Assignment,Expression
Expand Down
Loading