"imports":"""

""",

"classes": """

""",

"functions": """

""",

"calls": """

""",

"variables": """

""",

"lambda_assignments": """

""",

def __init__(self, lang_name: str):

# myself

self.lang_name = lang_name

# -trans

self.lang: Language = get_language(lang_name)

# -setup-conf

self.parser = Parser()

self.parser.set_language(self.lang)

# to PythonParser,it's a enhance

self.lang_parser = PythonParser(self)

def create_func_class_table(self, files: list[Path]) -> dict:

table = {}

query_str = """

query = self.lang.query(query_str)

for file_path in files:

with open(file_path, 'r', encoding='utf-8') as f:

code = f.read()

tree = self.parser.parse(bytes(code, 'utf-8'))

root_node = tree.root_node

for node, tag in query.captures(root_node):

if tag == 'name':

name = node.text.decode('utf-8')

if name not in table:

table[name] = []

# Normalize path to use forward slashes for consistency

normalized_path = str(file_path.resolve()).replace('\\', '/')

table[name].append(normalized_path)

return table

def parse_file(self, repo_path, file_path: Path, is_dependency: bool = False) -> dict:

'''Parse a single file and return its metadata. add a new field 'repo_path' to file_data'''

file_data = self.lang_parser.parse(file_path, is_dependency=is_dependency)

file_data['repo_path'] = str(repo_path)

debug_log(f"Parsed file: {file_path}, found {len(file_data.get('functions', []))} functions and {len(file_data.get('classes', []))} classes.")

def __init__(self, parser_wrapper):

# from parser_wrapper, a inherit somehow

self.parser_wrapper = parser_wrapper

self.lang_name = parser_wrapper.lang_name

self.lang = parser_wrapper.lang

self.parser = parser_wrapper.parser

# myself

print(f"debug:queries init, maybe leak a arg {self.lang.language_id}")

self.queries = {name: self.lang.query(query_str) for name, query_str in PY_QUREISE.items()}

def _get_node_text(self, node):

return node.text.decode('utf-8')

def _get_parent_context(self, node, types=('function_definition','class_definition')):

# walkup from arg:node

curr = node.parent

while curr:

if curr.type in types:

curr_in_type_child_is_name = curr.child_by_field_name('name')

return (

self._get_node_text(curr_in_type_child_is_name) if curr_in_type_child_is_name else None,

curr.type,

curr.start_point[0] + 1

)

curr = curr.parent

return None, None, None

def _get_docstring(self, body_node):

if body_node and body_node.child_count > 0:

first_child = body_node.children[0]

if first_child.type == 'expression_statement' and first_child.children[0].type == 'string':

try:

return ast.literal_eval(self._get_node_text(first_child.children[0]))

except (ValueError, SyntaxError):

return self._get_node_text(first_child.children[0])

return None

def _find_imports(self, root_node):

# local var

imports = []

picked_imports = set()

query = self.queries['imports']

# loop

for node, tag in query.captures(root_node):

import_meta = None

if tag in ('import', 'from_import_stmt'):

# For import

if tag == 'import':

node_text = self._get_node_text(node)

# about alias

alias = None

if ' as ' in node_text:

# so we have a alias

fullname_and_alias = node_text.split(' as ')

fullname = fullname_and_alias[0]

alias = fullname_and_alias[1]

else:

# so we dont have a alias

fullname = node_text.strip()

# about imports module record

if fullname in picked_imports:

# we have picked it

continue

else:

picked_imports.add(fullname)

# fill dataclass

import_meta = {

"name": fullname,

"full_import_name": fullname,

"start_line": node.start_point[0] + 1,

"alias": alias,

"context": self._get_parent_context(node)[:2],

"lang": self.lang_name,

"is_dependency": False

}

imports.append(import_meta)

# For from_import_statement

elif tag == 'from_import_stmt':

# For import .. from <>

subnode_module = node.child_by_field_name('module_name')

subnode_module_text = self._get_node_text(subnode_module)

if not subnode_module: continue

# For import <> from ..

subnode_be_imported = node.child_by_field_name('name')

if subnode_be_imported:

for child in subnode_be_imported.children:

child_name_node_text = None

child_alias_node_text = None

if child.type == 'aliased_import':

child_name_node = child.child_by_field_name('name')

child_alias_node = child.child_by_field_name('alias')

if child_name_node:

child_name_node_text = self._get_node_text(child_name_node)

if child_alias_node:

child_alias_node_text = self._get_node_text(child_alias_node)

elif child.type == 'dotted_name' or child.type == 'identifier':

child_name_node_text = self._get_node_text(child)

# Only create import_meta if we successfully extracted the name

if child_name_node_text:

child_fullname = f"{subnode_module_text}.{child_name_node_text}"

if child_fullname in picked_imports:

continue

else:

picked_imports.add(child_fullname)

# fill dataclass

import_meta = {

"name": child_name_node_text,

"full_import_name": child_fullname,

"start_line": node.start_point[0] + 1,

"alias": child_alias_node_text,

"context": self._get_parent_context(child)[:2],

"lang": self.lang_name,

"is_dependency": False

}

imports.append(import_meta)

return imports

def _find_classes(self, root_node):

# local var

classes = []

query = self.queries['classes']

# loop

for node, tag in query.captures(root_node):

class_meta = None

if tag == 'name':

class_name = self._get_node_text(node)

class_node = node.parent

class_body_node = class_node.child_by_field_name('body')

# superclasses

base_classes_node = class_node.child_by_field_name('superclasses')

base_classes = []

if base_classes_node:

for child in base_classes_node.children:

if child.type == 'identifier' or child.type == 'attribute':

base_classes.append(self._get_node_text(child))

# context

context,_,_ = self._get_parent_context(class_node)

# decorator - check if parent is decorated_definition

decorators = []

if class_node.parent and class_node.parent.type == 'decorated_definition':

decorators = [self._get_node_text(child) for child in class_node.parent.children if child.type == 'decorator']

# fill metadata

class_meta = {

"name": class_name,

"source": self._get_node_text(class_node),

"decorators": decorators,

"base_classes": base_classes,

"start_line": class_node.start_point[0] + 1,

"end_line": class_node.end_point[0] + 1,

"docstring": self._get_docstring(class_body_node),

"context": context,

"lang": self.lang_name,

"is_dependency": False

}

classes.append(class_meta)

return classes

def _find_functions(self, root_node):

functions = []

query = self.queries['functions']

for node, tag in query.captures(root_node):

function_meta = None

if tag == 'name':

function_name = self._get_node_text(node)

function_node = node.parent

function_body_node = function_node.child_by_field_name('body')

# parameters

parameters_node = function_node.child_by_field_name('parameters')

parameters = []

if parameters_node:

for child in parameters_node.children:

param_text = None

if child.type == 'identifier':

param_text = self._get_node_text(child)

elif child.type == 'default_parameter':

param_name_node = child.child_by_field_name('name')

if param_name_node:

param_text = self._get_node_text(param_name_node)

elif child.type == 'typed_parameter':

param_name_node = child.child_by_field_name('name')

if param_name_node:

param_text = self._get_node_text(param_name_node)

elif child.type == 'typed_default_parameter':

param_name_node = child.child_by_field_name('name')

if param_name_node:

param_text = self._get_node_text(param_name_node)

if param_text:

parameters.append(param_text)

# return type

return_type_node = function_node.child_by_field_name('return_type')

return_type = None

if return_type_node:

return_type = self._get_node_text(return_type_node)

# context

context,context_type,_ = self._get_parent_context(function_node)

class_context,_,_ = self._get_parent_context(function_node,types=('class_definition',))

# decorator - check if parent is decorated_definition

decorators = []

if function_node.parent and function_node.parent.type == 'decorated_definition':

decorators = [self._get_node_text(child) for child in function_node.parent.children if child.type == 'decorator']

# fill metadata

function_meta = {

"name": function_name,

"source": self._get_node_text(function_node),

"decorators": decorators,

"parameters": parameters,

"return_type": return_type,

"start_line": function_node.start_point[0] + 1,

"end_line": function_node.end_point[0] + 1,

"docstring": self._get_docstring(function_body_node),

"context": context,

"context_type": context_type,

"class_context": class_context,

"lang": self.lang_name,

"is_dependency": False

}

functions.append(function_meta)

return functions

def _find_calls(self, root_node):

calls = []

query = self.queries['calls']

for node, tag in query.captures(root_node):

call_meta = None

if tag == 'name':

call_name = self._get_node_text(node)

call_node = node.parent if node.parent.type == 'call' else node.parent.parent

full_call_node = call_node.child_by_field_name('function')

# params

params = []

params_node = call_node.child_by_field_name('arguments')

if params_node:

params_text = [self._get_node_text(child) for child in params_node.children]

if params_text:

params.append(params_text)

# context (function or class) ; no long limit

context = self._get_parent_context(node)

# fill metadata

call_meta = {

"name": call_name,

"full_name": self._get_node_text(full_call_node),

"parameters": params,

"start_line": node.start_point[0] + 1,

"end_line": node.end_point[0] + 1,

"context": context,

"class_context": self._get_parent_context(call_node, types=('class_definition',))[:2],

"lang": self.lang_name,

"is_dependency": False

}

calls.append(call_meta)

return calls

def _find_variables(self, root_node):

variables = []

query = self.queries['variables']

for node, tag in query.captures(root_node):

variable_meta = None

if tag == 'name':

variable_name = self._get_node_text(node)

assignment_node = node.parent

right_node = assignment_node.child_by_field_name('right')

if right_node and right_node.type == 'lambda':

# skip lambda assignments

continue

value = self._get_node_text(right_node) if right_node else None

type_node = assignment_node.child_by_field_name('type')

type_text = self._get_node_text(type_node) if type_node else None

# context (function or class) ; no long limit

context,_,_ = self._get_parent_context(node)

class_context,_,_ = self._get_parent_context(node, types=('class_definition',))

# fill metadata

variable_meta = {

"name": variable_name,

"value": value,

"type": type_text,

"start_line": node.start_point[0] + 1,

"end_line": node.end_point[0] + 1,

"context": context,

"class_context": class_context,

"lang": self.lang_name,

"is_dependency": False

}

variables.append(variable_meta)

return variables

def _find_lambda_assignments(self, root_node):

lambda_assignments = []

query = self.queries['lambda_assignments']

for node, tag in query.captures(root_node):

lambda_meta = None

if tag == 'name':

variable_name = self._get_node_text(node)

assignment_node = node.parent

lambda_node = assignment_node.child_by_field_name('right')

params_node = lambda_node.child_by_field_name('parameters')

# context (function or class) ; no long limit

context,context_type,_ = self._get_parent_context(assignment_node)

class_context,_,_ = self._get_parent_context(assignment_node, types=('class_definition',))

# fill metadata

lambda_meta = {

"name": variable_name,

"source": self._get_node_text(assignment_node),

"start_line": node.start_point[0] + 1,

"end_line": assignment_node.end_point[0] + 1,

"context": context,

"context_type": context_type,

"class_context": class_context,

"lang": self.lang_name,

"is_dependency": False

}

lambda_assignments.append(lambda_meta)

return lambda_assignments

def parse(self, file_path: Path, is_dependency: bool = False) -> dict:

# read file content

with open(file_path, 'r', encoding='utf-8') as f:

code = f.read()

# parse to tree

tree = self.parser.parse(bytes(code, 'utf-8'))

root_node = tree.root_node

# extract metadata

imports = self._find_imports(root_node)

classes = self._find_classes(root_node)

functions = self._find_functions(root_node)

functions.extend(self._find_lambda_assignments(root_node))

calls = self._find_calls(root_node)

variables = self._find_variables(root_node)

# return all metadata

return {

"file_path": str(file_path),

"imports": imports,

"classes": classes,

"functions": functions,

"function_calls": calls,

"variables": variables,

"is_dependency": is_dependency,

"lang": self.lang_name