forked from PyGithub/PyGithub
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathopenapi.py
More file actions
3198 lines (2874 loc) · 140 KB
/
Copy pathopenapi.py
File metadata and controls
3198 lines (2874 loc) · 140 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
############################ Copyrights and license ############################
# #
# Copyright 2025 Enrico Minack <[email protected]> #
# #
# This file is part of PyGithub. #
# http://pygithub.readthedocs.io/ #
# #
# PyGithub is free software: you can redistribute it and/or modify it under #
# the terms of the GNU Lesser General Public License as published by the Free #
# Software Foundation, either version 3 of the License, or (at your option) #
# any later version. #
# #
# PyGithub is distributed in the hope that it will be useful, but WITHOUT ANY #
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS #
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more #
# details. #
# #
# You should have received a copy of the GNU Lesser General Public License #
# along with PyGithub. If not, see <http://www.gnu.org/licenses/>. #
# #
################################################################################
from __future__ import annotations
import abc
import argparse
import dataclasses
import difflib
import json
import os.path
import re
import sys
from collections import Counter, defaultdict
from collections.abc import Callable
from enum import Enum
from json import JSONEncoder
from os import listdir
from os.path import isfile, join
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, Sequence
import libcst as cst
import requests
from libcst import Expr, IndentedBlock, Module, SimpleStatementLine, SimpleString
equal = cst.AssignEqual(cst.SimpleWhitespace(""), cst.SimpleWhitespace(""))
def resolve_schema(schema_type: dict[str, Any], spec: dict[str, Any]) -> dict[str, Any]:
if "$ref" in schema_type:
schema = schema_type.get("$ref").strip("# ")
ref_schema_type = spec
for step in schema.split("/"):
if step:
if step not in ref_schema_type:
raise ValueError(f"Could not find schema in spec: {schema}")
ref_schema_type = ref_schema_type[step]
return ref_schema_type
return schema_type
def as_python_type(
schema_type: dict[str, Any],
schema_path: list[str],
schema_to_class: dict[str, str],
classes,
*,
verbose: bool = False,
collect_new_schemas: list[str] | None = None,
) -> PythonType | GithubClass | None:
schema = None
data_type = schema_type.get("type")
if "$ref" in schema_type:
schema = schema_type.get("$ref").strip("# ")
elif "oneOf" in schema_type:
types = [
as_python_type(
t,
schema_path + ["oneOf", str(idx)],
schema_to_class,
classes,
verbose=verbose,
collect_new_schemas=collect_new_schemas,
)
for idx, t in enumerate(schema_type.get("oneOf"))
]
types = list({t for t in types if t is not None})
if len(types) == 0:
return None
if len(types) == 1:
return types[0]
return PythonType("union", sorted(types))
elif "allOf" in schema_type and len(schema_type.get("allOf")) == 1:
return as_python_type(
schema_type.get("allOf")[0],
schema_path + ["allOf", "0"],
schema_to_class,
classes,
verbose=verbose,
collect_new_schemas=collect_new_schemas,
)
if data_type == "object":
schema = "/".join([""] + schema_path)
if schema is not None:
# these schemas are explicitly ignored
if schema in {"/components/schemas/empty-object"}:
return None
if schema in schema_to_class:
classes_of_schema = schema_to_class[schema]
if not isinstance(classes_of_schema, list):
raise ValueError(f"Expected list of types for schema: {schema}")
if len(classes_of_schema) == 0:
raise ValueError(f"Expected non-empty list of types for schema: {schema}")
if len(classes_of_schema) == 1:
class_name = classes_of_schema[0]
if class_name not in classes:
if verbose:
print(f"Class not found in index: {class_name}")
return None
return GithubClass(**classes.get(class_name))
if verbose:
for class_name in classes:
if class_name not in classes:
print(f"Class not found in index: {class_name}")
return PythonType(
type="union",
inner_types=[GithubClass(**classes.get(cls)) for cls in sorted(classes_of_schema) if cls in classes],
)
if collect_new_schemas is not None:
collect_new_schemas.append(schema or ".".join([""] + schema_path))
if verbose:
print(f"Schema not implemented: {schema or '.'.join([''] + schema_path)}")
return PythonType(type="dict", inner_types=[PythonType("str"), PythonType("Any")])
if data_type is None:
if verbose:
print(f"There is no $ref and no type in schema: {json.dumps(schema_type)}")
return None
if data_type == "array":
return PythonType(
type="list",
inner_types=[
as_python_type(
schema_type.get("items"),
schema_path + ["items"],
schema_to_class,
classes,
verbose=verbose,
collect_new_schemas=collect_new_schemas,
)
],
)
format = schema_type.get("format")
data_types = {
"boolean": {None: "bool"},
"integer": {None: "int"},
"number": {None: "float"},
"string": {
None: "str",
"date-time": "datetime",
"uri": "str",
},
}
if data_type not in data_types:
if verbose:
print(f"Unsupported data type: {data_type}")
return None
formats = data_types.get(data_type)
return PythonType(type=formats.get(format) or formats.get(None))
@dataclasses.dataclass(frozen=True)
class PythonType:
type: str
inner_types: list[PythonType | GithubClass] | None = None
def __hash__(self):
return hash(self.__repr__())
def __repr__(self):
return (
f"{self.type}[{', '.join([str(inner) for inner in self.inner_types])}]" if self.inner_types else self.type
)
def __lt__(self, other) -> bool:
return self.__repr__() < other.__repr__()
@dataclasses.dataclass(frozen=True)
class GithubClass:
ids: list[str]
package: str
module: str
name: str
filename: str
test_filename: str
bases: list[str]
inheritance: list[str]
methods: dict
properties: dict
schemas: list[str]
docstring: str
def __hash__(self):
return hash(self.__repr__())
def __repr__(self):
return ".".join([self.package, self.module, self.name])
def __lt__(self, other) -> bool:
return self.__repr__() < other.__repr__()
@property
def short_class_name(self) -> str:
return self.name.split(".")[-1]
@property
def full_class_name(self) -> str:
return f"{self.package}.{self.module}.{self.name}"
@staticmethod
def from_class_name(
class_name: str, index: dict[str, Any] | None = None, github_parent_path: str = ""
) -> GithubClass:
if github_parent_path and not github_parent_path.endswith("/"):
github_parent_path = f"{github_parent_path}/"
if "." in class_name:
full_class_name = class_name
package, module, class_name = full_class_name.split(".", 2)
if index is not None:
clazz = GithubClass.from_class_name(class_name, index)
if clazz.package != package or clazz.module != module or clazz.name != class_name:
raise ValueError(f"Class mismatch: {full_class_name} vs {clazz}")
return clazz
else:
return GithubClass(
ids=[],
package="github",
module=class_name,
name=class_name,
filename=f"{github_parent_path}{package}/{module}.py",
test_filename=f"{github_parent_path}tests/{module}.py",
bases=[],
inheritance=[],
methods={},
properties={},
schemas=[],
docstring="",
)
else:
if index is not None:
classes = index.get("classes", {})
if class_name not in classes:
raise ValueError(f"Unknown class {class_name}")
cls = classes.get(class_name)
if any(key not in cls for key in ["package", "module", "name"]):
raise KeyError(f"Missing package, module or name in {cls}")
return GithubClass(**cls)
else:
return GithubClass.from_class_name(
f"github.{class_name}.{class_name}", github_parent_path=github_parent_path
)
@dataclasses.dataclass(frozen=True)
class Property:
name: str
data_type: PythonType | GithubClass | None
deprecated: bool
class SimpleStringCollector(cst.CSTVisitor):
def __init__(self):
super().__init__()
self._strings = []
@property
def strings(self):
return self._strings
def visit_SimpleString(self, node: cst.SimpleString) -> bool | None:
self._strings.append(node.evaluated_value)
class FunctionCallCollector(cst.CSTVisitor):
def __init__(self):
super().__init__()
self._calls = []
@property
def calls(self):
return self._calls
def visit_Call(self, node: cst.Call) -> bool | None:
code = cst.Module([]).code_for_node
func_name = code(node.func)
args = [(f"{code(arg.keyword)}=" if arg.keyword else "") + code(arg.value) for arg in node.args]
self._calls.append((func_name, args))
def get_class_docstring(node: cst.ClassDef) -> str | None:
try:
if (
isinstance(node.body, IndentedBlock)
and isinstance(node.body.body[0], SimpleStatementLine)
and isinstance(node.body.body[0].body[0], Expr)
and isinstance(node.body.body[0].body[0].value, SimpleString)
):
return node.body.body[0].body[0].value.value
except Exception as e:
print(f"Extracting docstring of class {node.name.value} failed", e)
def merge_paths(paths: list[dict[str, Any]]) -> dict[str, Any]:
merged_paths = {}
for path_dict in paths:
for path, verbs in path_dict.items():
for verb, methods in verbs.items():
if path not in merged_paths:
merged_paths[path] = {}
if verb not in merged_paths[path]:
merged_paths[path][verb] = {}
if "methods" not in merged_paths[path][verb]:
merged_paths[path][verb]["methods"] = []
merged_paths[path][verb]["methods"].extend(methods.get("methods", []))
return merged_paths
class CstMethods(abc.ABC):
@staticmethod
def contains_decorator(seq: Sequence[cst.Decorator], decorator_name: str):
return any(d.decorator.value == decorator_name for d in seq if isinstance(d.decorator, cst.Name))
@classmethod
def is_github_object_property(cls, func_def: cst.FunctionDef):
return cls.contains_decorator(func_def.decorators, "property")
@classmethod
def create_subscript(cls, name: str) -> cst.Subscript:
fields = name.rstrip("]").split("[", maxsplit=1)
name = fields[0]
index = fields[1]
sub = cst.Subscript(cst.Name(name), [cst.SubscriptElement(cst.Index(cst.Integer(index)))])
return sub
@classmethod
def create_attribute(cls, names: list[str]) -> cst.BaseExpression:
names = [cls.create_subscript(name) if "[" in name and name.endswith("]") else cst.Name(name) for name in names]
if len(names) == 1:
return names[0]
attr = cst.Attribute(names[0], names[1])
for name in names[2:]:
attr = cst.Attribute(attr, name)
return attr
@classmethod
def create_type(
cls, data_type: PythonType | GithubClass | None, short_class_name: bool = False
) -> cst.BaseExpression:
if data_type is None:
return cst.Name("None")
if isinstance(data_type, GithubClass):
if short_class_name:
return cst.Name(data_type.name.split(".")[-1])
return cls.create_attribute([data_type.package, data_type.module] + data_type.name.split("."))
if data_type.type == "union":
if len(data_type.inner_types) == 0:
return cst.Name("None")
if len(data_type.inner_types) == 1:
return cls.create_type(data_type.inner_types[0], short_class_name)
result = cst.BinaryOperation(
cls.create_type(data_type.inner_types[0], short_class_name),
cst.BitOr(),
cls.create_type(data_type.inner_types[1], short_class_name),
)
for dt in data_type.inner_types[2:]:
result = cst.BinaryOperation(result, cst.BitOr(), cls.create_type(dt, short_class_name))
return result
if data_type.inner_types:
elems = [
cst.SubscriptElement(cst.Index(cls.create_type(elem, short_class_name)))
for elem in data_type.inner_types
]
return cst.Subscript(cst.Name(data_type.type), slice=elems)
return cst.Name(data_type.type)
@classmethod
def find_nodes(cls, node: cst.CSTNode, node_type: type[cst.CSTNode]) -> list[cst.CSTNode]:
if isinstance(node, node_type):
return [node]
return [node for child in node.children for node in cls.find_nodes(child, node_type)]
@staticmethod
def parse_attribute(attr: cst.Attribute) -> list[str]:
attrs = []
while (
isinstance(attr, cst.Attribute) or isinstance(attr, cst.Subscript) and isinstance(attr.value, cst.Attribute)
):
if isinstance(attr, cst.Attribute):
attrs.insert(0, attr.attr.value)
elif isinstance(attr, cst.Subscript):
# we do not extract a name, we skip to the subscript value
pass
attr = attr.value
attrs.insert(0, attr.value)
return attrs
class CstVisitorBase(cst.CSTVisitor, CstMethods, abc.ABC):
def __init__(self):
super().__init__()
self.visit_class_name = []
def visit_ClassDef(self, node: cst.ClassDef):
self.visit_class_name.append(node.name.value)
def leave_ClassDef(self, original_node: cst.ClassDef) -> None:
self.visit_class_name.pop()
@property
def current_class_name(self) -> str:
return ".".join(self.visit_class_name)
class CstTransformerBase(cst.CSTTransformer, CstMethods, abc.ABC):
def __init__(self):
super().__init__()
self.visit_class_name = []
def visit_ClassDef(self, node: cst.ClassDef):
self.visit_class_name.append(node.name.value)
def leave_ClassDef(self, original_node: cst.ClassDef, updated_node: cst.ClassDef):
self.visit_class_name.pop()
return super().leave_ClassDef(original_node, updated_node)
@property
def current_class_name(self) -> str:
return ".".join(self.visit_class_name)
@staticmethod
def is_github_import(stmt: cst.Import | cst.ImportFrom) -> bool:
return (
isinstance(stmt, cst.Import)
and (
isinstance(stmt.names[0].name, cst.Name)
and stmt.names[0].name.value == "github"
or isinstance(stmt.names[0].name, cst.Attribute)
and stmt.names[0].name.value.value == "github"
)
or isinstance(stmt, cst.ImportFrom)
and isinstance(stmt.module, cst.Attribute)
and stmt.module.value.value == "github"
)
@staticmethod
def is_datetime_import(stmt: cst.Import | cst.ImportFrom) -> bool:
return (
isinstance(stmt, cst.ImportFrom)
and isinstance(stmt.module, cst.Name)
and stmt.module.value == "datetime"
and stmt.names
and isinstance(stmt.names[0], cst.ImportAlias)
and isinstance(stmt.names[0].name, cst.Name)
and stmt.names[0].name.value == "datetime"
)
@staticmethod
def add_datetime_import(node: cst.Module, index: int) -> cst.Module:
import_stmt = cst.SimpleStatementLine(
[
cst.ImportFrom(
cst.Name("datetime"), [cst.ImportAlias(cst.Name("datetime")), cst.ImportAlias(cst.Name("timezone"))]
)
]
)
stmts = list(node.body)
return node.with_changes(body=stmts[:index] + [import_stmt] + stmts[index:])
@staticmethod
def add_future_import(node: cst.Module) -> cst.Module:
stmts = list(node.body)
first_stmt = stmts[0] if stmts else None
if not (
first_stmt
and isinstance(first_stmt, cst.SimpleStatementLine)
and isinstance(first_stmt.body[0], cst.ImportFrom)
and isinstance(first_stmt.body[0].module, cst.Name)
and first_stmt.body[0].module.value == "__future__"
and first_stmt.body[0].names
and isinstance(first_stmt.body[0].names[0], cst.ImportAlias)
and isinstance(first_stmt.body[0].names[0].name, cst.Name)
and first_stmt.body[0].names[0].name.value == "annotations"
):
import_stmt = cst.SimpleStatementLine(
[cst.ImportFrom(cst.Name("__future__"), [cst.ImportAlias(cst.Name("annotations"))])]
)
if (
isinstance(first_stmt, cst.SimpleStatementLine)
and isinstance(first_stmt.body[0], (cst.Import, cst.ImportFrom))
and not first_stmt.leading_lines
):
first_stmt = first_stmt.with_changes(leading_lines=[cst.EmptyLine()])
stmts = [first_stmt] + stmts[1:]
node = node.with_changes(body=[import_stmt] + stmts)
return node
class IndexPythonClassesVisitor(CstVisitorBase):
def __init__(self, classes: dict[str, Any], paths: dict[str, Any], method_verbs: dict[str, str] | None):
super().__init__()
self._module = None
self._package = None
self._filename = None
self._test_filename = None
self._classes = classes
self._paths = paths
self._ids = []
self._properties = {}
self._methods = {}
self._method_verbs = method_verbs
def module(self, module: str):
self._module = module
def package(self, package: str):
self._package = package
def filename(self, filename: str):
self._filename = filename
def test_filename(self, test_filename: str):
self._test_filename = test_filename
@property
def classes(self) -> dict[str, Any]:
return self._classes
def leave_ClassDef(self, node: cst.ClassDef) -> bool | None:
class_name = self.current_class_name
class_name_short = node.name.value
class_docstring = get_class_docstring(node)
class_docstring = class_docstring.strip('"\r\n ') if class_docstring else None
class_schemas = []
class_bases = [
val if isinstance(val, str) else Module([]).code_for_node(val).split(".")[-1]
for base in node.bases
for val in [base.value.value]
]
# extract OpenAPI schema
if class_docstring:
lines = class_docstring.splitlines()
for idx, line in enumerate(lines):
if "The OpenAPI schema can be found at" in line:
while len(lines) > idx + 1 and not lines[idx + 1].strip():
idx = idx + 1
for schema in lines[idx + 1 :]:
if not schema.strip().lstrip("- "):
break
class_schemas.append(schema.strip().lstrip("- "))
if class_name_short in self._classes:
print(f"Duplicate class definition for {class_name_short}")
# TODO: ideally, the key should be the fully qualified class name and there
# should be an index from class_name to the fully qualified class name
self._classes[class_name_short] = {
"ids": self._ids,
"name": class_name,
"module": self._module,
"package": self._package,
"filename": self._filename,
"test_filename": self._test_filename,
"docstring": class_docstring,
"schemas": class_schemas,
"bases": class_bases,
"properties": self._properties,
"methods": self._methods,
}
self._ids = []
self._properties = {}
self._methods = {}
return super().leave_ClassDef(node)
@staticmethod
def return_types(return_type: str | None) -> list[str]:
if return_type is None:
return []
return_type = return_type.strip()
none = []
if return_type.startswith("None | "):
none = ["None"]
return_type = return_type[7:]
elif return_type.endswith("| None"):
none = ["None"]
return_type = return_type[:-7]
types = [return_type] + none
if "|" in return_type and "[" not in return_type:
types = return_type.split("|") + none
return [rt.strip().replace('"', "") for rt in types]
def visit_FunctionDef(self, node: cst.FunctionDef) -> None:
method_name = node.name.value
returns = self.return_types(cst.Module([]).code_for_node(node.returns.annotation) if node.returns else None)
if self.is_github_object_property(node):
self._properties[method_name] = {"name": method_name, "returns": returns}
visitor = SimpleStringCollector()
node.body.visit(visitor)
if visitor.strings:
string = [line for line in visitor.strings[0].splitlines() if ":calls:" in line]
if string:
fields = string[0].split(":calls:")[1].strip(" `").split(" ", maxsplit=2)
self._methods[method_name] = {
"name": method_name,
"call": {
"verb": fields[0] if len(fields) > 0 else None,
"path": fields[1] if len(fields) > 1 else None,
"docs": fields[2] if len(fields) > 2 else None,
},
"returns": returns,
}
if len(fields) > 1:
verb = fields[0]
path = fields[1]
if path not in self._paths:
self._paths[path] = {}
if verb not in self._paths[path]:
self._paths[path][verb] = {"methods": []}
self._paths[path][verb]["methods"].append(
{
"class": self.current_class_name,
"name": method_name,
"returns": returns,
}
)
# check if method (VERB) is same as in the code
if len(fields) > 0 and self._method_verbs is not None:
verb = f'"{fields[0]}"'
full_method_name = f"{self.current_class_name}.{method_name}"
if full_method_name in self._method_verbs:
# these are methods configured in the github/openapi.index.json file
known_verb = f'"{self._method_verbs[full_method_name]}"'
if known_verb != verb:
print(
f"Method {full_method_name} is known to call {known_verb}, "
f"but doc-string says {verb}"
)
else:
# detect method from code
visitor = FunctionCallCollector()
node.body.visit(visitor)
calls = visitor.calls
# calls to PaginatedList(...) are equivalent to
# self.__requester.requestJsonAndCheck("GET", …, parameters=…, headers=…)
calls = [
("self.__requester.requestJsonAndCheck", ['"GET"', "…", "parameters=…", "headers=…"])
if func in ["PaginatedList", "github.PaginatedList.PaginatedList"]
else (func, args)
for func, args in calls
]
# calls to self._requester.graphql_ are equivalent to
# self._requester.requestJsonAndCheck("POST", …, input=…)
calls = [
("self._requester.requestJsonAndCheck", ['"POST"', "…", "input=…"])
if func.startswith("self._requester.graphql_")
else (func, args)
for func, args in calls
]
# calls to github.AuthenticatedUser.AuthenticatedUser(self.__requester, url=url, completed=False)
# where class extends CompletableGithubObject are equivalent to
# self._requester.requestJsonAndCheck("GET", …, headers=…)
calls = [
(
"self._requester.requestJsonAndCheck",
['"GET"', "…", "headers=…"],
"CompletableGithubObject",
)
if func.startswith("github.")
and args
and args[0]
in [
"self._requester",
"self.__requester",
"requester=self._requester",
"requester=self.__requester",
]
and (
len(args) > 1
and args[1].startswith("url=")
or len(args) > 2
and args[2].startswith(('{"url":', 'attributes={"url":'))
)
else (func, args, None)
for func, args in calls
]
# skip functions that call into parent functions
if not any(func.startswith("super().") for func, args, base in calls):
# check for requester calls with the expected verb
if not any(
func.startswith(("self._requester.request", "self.__requester.request"))
and args
and args[0] == verb
for func, args, base in calls
):
print(f"Not found any {verb} call in {self.current_class_name}.{method_name}")
for func, args, base in calls:
print(f"- calls {func}({', '.join(args)})")
# else:
# # check if the found verb depends on a base class, which we cannot test here
# if not any(
# func.startswith(("self._requester.request", "self.__requester.request"))
# and args
# and args[0] == verb
# and base is None
# for func, args, base in calls
# ):
# print(
# f"Not found any {verb} call in {self.current_class_name}.{method_name} "
# f"conditional on some base class"
# )
# for func, args, base in calls:
# print(f"- calls {func}({', '.join(args)})")
if method_name == "__repr__":
# extract properties used here as ids
visitor = DictKeyCollector(self._ids)
node.visit(visitor)
class DictKeyCollector(cst.CSTVisitor):
def __init__(self, keys: list[str]):
super().__init__()
self.keys = keys
def visit_DictElement_key(self, node: cst.DictElement):
self.keys.append(node.key.value.strip('"'))
class ApplySchemaBaseTransformer(CstTransformerBase, abc.ABC):
def __init__(
self,
module_name: str,
class_name: str,
properties: dict[str, (PythonType | GithubClass | None, bool)],
deprecate: bool,
):
super().__init__()
self.module_name = module_name
self.class_name = class_name
properties = [Property(name=n, data_type=t, deprecated=d) for n, (t, d) in properties.items()]
self.properties = sorted(properties, key=lambda p: p.name)
self.all_properties = self.properties.copy()
self.deprecate = deprecate
@property
def current_property(self) -> Property | None:
if not self.properties:
return None
return self.properties[0]
class ApplySchemaTransformer(ApplySchemaBaseTransformer):
def __init__(
self,
module_name: str,
class_name: str,
properties: dict[str, (PythonType | GithubClass | None, bool)],
completable: bool,
deprecate: bool,
):
super().__init__(module_name, class_name, properties, deprecate)
self.completable = completable
@staticmethod
def deprecate_function(node: cst.FunctionDef) -> cst.FunctionDef:
decorators = list(node.decorators)
decorators.append(cst.Decorator(decorator=cst.Name(value="deprecated")))
return node.with_changes(decorators=decorators)
def inner_github_type(self, data_type: PythonType | GithubClass | list[PythonType | GithubClass]) -> [GithubClass]:
if data_type is None:
return []
if isinstance(data_type, list):
return [ght for dt in data_type for ght in self.inner_github_type(dt)]
if isinstance(data_type, PythonType):
return self.inner_github_type(data_type.inner_types)
if isinstance(data_type, GithubClass):
return [data_type]
raise ValueError("Unsupported data type", data_type)
def leave_Module(self, original_node: Module, updated_node: Module) -> Module:
i = 0
node = updated_node
# add from __future__ import annotations if not the first import
node = self.add_future_import(node)
property_classes = {
ghc
for p in self.all_properties
for ghc in self.inner_github_type(p.data_type)
if ghc.module != self.module_name and ghc.name != self.class_name
}
import_classes = sorted(property_classes, key=lambda c: c.module)
typing_classes = sorted(property_classes, key=lambda c: c.module)
# TODO: do not import this file itself
datetime_exists = False
in_github_imports = False
needs_datetime_import = any(
p.data_type.type == "datetime" for p in self.all_properties if isinstance(p.data_type, PythonType)
)
# insert import classes if needed
while (
i < len(node.body)
and isinstance(node.body[i], cst.SimpleStatementLine)
and isinstance(node.body[i].body[0], (cst.Import, cst.ImportFrom))
):
if self.is_datetime_import(node.body[i].body[0]):
datetime_exists = True
if not in_github_imports and self.is_github_import(node.body[i].body[0]):
in_github_imports = True
# emit datetime import if needed
if needs_datetime_import and not datetime_exists:
node = self.add_datetime_import(node, i)
datetime_exists = True
i = i + 1
if in_github_imports and import_classes:
import_node = node.body[i].body[0]
imported_module = (
(
import_node.module.value
if isinstance(import_node.module, cst.Name)
else import_node.module.attr.value
)
if isinstance(import_node, cst.ImportFrom)
else import_node.names[0].name.attr.value
)
while import_classes and import_classes[0].module < imported_module:
import_module = import_classes.pop(0)
import_stmt = cst.SimpleStatementLine(
[
cst.Import(
[cst.ImportAlias(self.create_attribute([import_module.package, import_module.module]))]
)
]
)
stmts = node.body
node = node.with_changes(body=tuple(stmts[:i]) + (import_stmt,) + tuple(stmts[i:]))
if import_classes and import_classes[0].module == imported_module:
import_classes.pop(0)
i = i + 1
# emit datetime import if needed
if needs_datetime_import and not datetime_exists:
node = self.add_datetime_import(node, i)
i = i + 1
while import_classes:
import_module = import_classes.pop(0)
import_stmt = cst.SimpleStatementLine(
[cst.Import([cst.ImportAlias(self.create_attribute([import_module.package, import_module.module]))])]
)
stmts = node.body
node = node.with_changes(body=tuple(stmts[:i]) + (import_stmt,) + tuple(stmts[i:]))
# insert typing classes if needed
# find first If statement in node.body
if_idx_node_or_none = next(
((idx, stmt) for idx, stmt in enumerate(node.body) if isinstance(stmt, cst.If)), None
)
if if_idx_node_or_none is not None:
if_idx, if_node = if_idx_node_or_none
i = 0
while i < len(if_node.body.body) and isinstance(if_node.body.body[i].body[0], (cst.Import, cst.ImportFrom)):
imported_module = if_node.body.body[i].body[0].module.attr.value
while typing_classes and typing_classes[0].module < imported_module:
typing_class = typing_classes.pop(0)
import_stmt = cst.SimpleStatementLine(
[
cst.ImportFrom(
module=self.create_attribute([typing_class.package, typing_class.module]),
names=[cst.ImportAlias(cst.Name(typing_class.name))],
)
]
)
stmts = if_node.body.body
if_node = if_node.with_changes(
body=if_node.body.with_changes(body=tuple(stmts[:i]) + (import_stmt,) + tuple(stmts[i:]))
)
if typing_classes and typing_classes[0].module == imported_module:
typing_classes.pop(0)
i = i + 1
while typing_classes:
typing_class = typing_classes.pop(0)
import_stmt = cst.SimpleStatementLine(
[
cst.ImportFrom(
module=self.create_attribute([typing_class.package, typing_class.module]),
names=[cst.ImportAlias(cst.Name(typing_class.name))],
)
]
)
stmts = if_node.body.body
if_node = if_node.with_changes(
body=if_node.body.with_changes(body=tuple(stmts[:i]) + (import_stmt,) + tuple(stmts[i:]))
)
node = node.with_changes(body=tuple(node.body[:if_idx]) + (if_node,) + tuple(node.body[if_idx + 1 :]))
return node
def leave_FunctionDef(self, original_node: cst.FunctionDef, updated_node: cst.FunctionDef):
if self.current_class_name != self.class_name:
return updated_node
if updated_node.name.value.startswith("__") and updated_node.name.value.endswith("__"):
return updated_node
if updated_node.name.value == "_initAttributes":
return self.update_init_attrs(updated_node)
nodes = []
if updated_node.name.value == "_useAttributes":
while self.current_property:
prop = self.properties.pop(0)
node = self.create_property_function(prop.name, prop.data_type, prop.deprecated)
nodes.append(cst.EmptyLine(indent=False))
nodes.append(node)
nodes.append(self.update_use_attrs(updated_node))
return cst.FlattenSentinel(nodes=nodes)
updated_node_is_github_object_property = self.is_github_object_property(updated_node)
while self.current_property and (
updated_node_is_github_object_property
and self.current_property.name < updated_node.name.value
or not updated_node_is_github_object_property
):
prop = self.properties.pop(0)
node = self.create_property_function(prop.name, prop.data_type, prop.deprecated)
nodes.append(cst.EmptyLine(indent=False))
nodes.append(node)
if updated_node_is_github_object_property:
if (
not self.current_property
or updated_node.name.value != self.current_property.name
or self.current_property.deprecated
):
nodes.append(self.deprecate_function(updated_node) if self.deprecate else updated_node)
else:
nodes.append(updated_node)
if self.current_property and updated_node.name.value == self.current_property.name:
self.properties.pop(0)
else:
nodes.append(updated_node)
return cst.FlattenSentinel(nodes=nodes)
def create_property_function(
self, name: str, data_type: PythonType | GithubClass | None, deprecated: bool
) -> cst.FunctionDef:
# we need to make the 'headers' attribute truly private,
# otherwise it conflicts with GithubObject._headers
attr_name = f"__{name}" if name == "headers" else f"_{name}"
complete_if_completable_stmt = cst.SimpleStatementLine(
body=[
cst.Expr(
cst.Call(
func=self.create_attribute(["self", "_completeIfNotSet"]),
args=[cst.Arg(self.create_attribute(["self", attr_name]))],
)
)
]