Skip to content

Develop #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Aug 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/codetext/parser/javascript_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,12 @@ def get_comment_node(function_node):

@staticmethod
def get_function_list(node):
function_types = ['function_declaration', 'function', 'method_definition', 'generator_function_declaration']
function_types = ['function_declaration',
'function',
'method_definition',
'generator_function_declaration',
'arrow_function',
'generator_function']
res = get_node_by_kind(node, function_types)
for node in res[:]:
if not node.children:
Expand Down Expand Up @@ -87,6 +92,16 @@ def get_function_metadata(function_node, blob: str=None) -> Dict[str, str]:
return_statement = get_node_by_kind(function_node, ['return_statement'])
if len(return_statement) > 0:
metadata['return_type'] = '<not_specific>'

if function_node.type in ["function",
"arrow_function",
"generator_function"]:
# function inside object property or variable declarator
identifier = function_node.prev_named_sibling
if identifier:
if identifier.type in ["identifier"]:
metadata["identifier"] = identifier.text.decode()

return metadata

@staticmethod
Expand Down
4 changes: 3 additions & 1 deletion src/codetext/parser/php_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ def get_comment_node(function_node):

@staticmethod
def get_class_list(node):
res = get_node_by_kind(node, ['class_declaration', 'trait_declaration'])
res = get_node_by_kind(node, ['class_declaration',
'trait_declaration',
'interface_declaration'])
return res

@staticmethod
Expand Down
4 changes: 3 additions & 1 deletion src/codetext/parser/python_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,9 @@ def get_class_metadata(class_node, blob: str=None) -> Dict[str, str]:
argument_list = get_node_text(child).split(',')
for arg in argument_list:
item = re.sub(r'[^a-zA-Z0-9\_]', ' ', arg).split()
metadata['parameters'][item[0].strip()] = None
# Handle class definitions with empty argument list class ABC()
if len(item) > 0:
metadata['parameters'][item[0].strip()] = None

# get __init__ function
return metadata
41 changes: 39 additions & 2 deletions src/codetext/parser/ruby_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ class RubyParser(LanguageParser):

@staticmethod
def get_function_list(node):
res = get_node_by_kind(node, ['method'])
res = get_node_by_kind(node, ['method',
'singleton_method'])
return res

@staticmethod
Expand Down Expand Up @@ -88,7 +89,7 @@ def get_function_metadata(function_node, blob=None) -> Dict[str, str]:
}

assert type(function_node) == tree_sitter.Node
assert function_node.type == 'method'
assert function_node.type in ['method', 'singleton_method']

for child in function_node.children:
if child.type == 'identifier':
Expand Down Expand Up @@ -133,3 +134,39 @@ def get_class_metadata(class_node, blob=None):
def get_comment_node(function_node):
comment_node = get_node_by_kind(function_node, kind='comment')
return comment_node

@staticmethod
def get_action_list(action_node):
call_nodes = get_node_by_kind(action_node, ['call'])
res = []
for call_node in call_nodes:
if get_node_by_kind(call_node, ["do_block"]):
res.append(call_node)
# print(res)
return res

@staticmethod
def get_action_metadata(action_node):
metadata = {
'identifier': '',
'parameters': {},
'return_type': None,
}

for child in action_node.children:
if child.type in ["identifier"]:
metadata['identifier'] = get_node_text(child)
if child.type in ["argument_list"]:
symbol = get_node_by_kind(child, ["simple_symbol"])
if symbol:
metadata['identifier'] += get_node_text(symbol[0])

parameters = get_node_by_kind(action_node, ["block_parameters"])

if parameters:
for param in get_node_by_kind(parameters[0], ["identifier"]):
param_name = get_node_text(param)
metadata['parameters'].update({param_name : None})

return metadata

55 changes: 53 additions & 2 deletions tests/test_parser/test_javascript.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ class Car {
def test_get_function_metadata(self):
root = self.root_node

function = JavascriptParser.get_function_list(root)[1]
metadata = JavascriptParser.get_function_metadata(function)
_function = JavascriptParser.get_function_list(root)[1]
metadata = JavascriptParser.get_function_metadata(_function)

for key in ['identifier', 'parameters', 'return_type']:
self.assertTrue(key in metadata.keys())
Expand Down Expand Up @@ -109,6 +109,57 @@ def test_get_class_metadata(self):
def test_extract_docstring(self):
pass


def test_metadata_with_arrow_function(self):
code_sample = '''
export const parseModel = async (mesh) =>
new Promise((resolve) => {
exporter.parse(
mesh,
(gltf) => {
const blob = new Blob([gltf], { type: "application/octet-stream" });
resolve(blob);
return blob;
},
(error) => {
console.log(error);
return error;

}
);
});
'''
root = parse_code(code_sample, 'javascript').root_node
fn = JavascriptParser.get_function_list(root)[0]
metadata = JavascriptParser.get_function_metadata(fn)

identifier = metadata['identifier']
self.assertEqual(identifier, 'parseModel')

def test_metadata_with_undecleared_functions(self):
code_sample = """
const asyncFunctionExpression = async function() {
// async function expression definition
return a
};

const generatorFunctionExpression = function*() {
// generator function expression definition
return b
};
"""
root = parse_code(code_sample, 'javascript').root_node
fn1, fn2 = JavascriptParser.get_function_list(root)

self.assertEqual(fn1.type, 'function')
self.assertEqual(fn2.type, 'generator_function')

metadata1 = JavascriptParser.get_function_metadata(fn1)
metadata2 = JavascriptParser.get_function_metadata(fn2)

self.assertEqual(metadata1['identifier'], 'asyncFunctionExpression')
self.assertEqual(metadata2['identifier'], 'generatorFunctionExpression')


if __name__ == '__main__':
unittest.main()
18 changes: 12 additions & 6 deletions tests/test_parser/test_php.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@ def test_get_function_list(self):

function_list = PhpParser.get_function_list(root)

self.assertEqual(len(function_list), 3)
self.assertEqual(len(function_list), 5)

def test_get_class_list(self):
root = self.root_node

class_list = PhpParser.get_class_list(root)

self.assertEqual(len(class_list), 1)
self.assertEqual(len(class_list), 3)

def test_get_docstring(self):
code_sample = """
Expand Down Expand Up @@ -104,11 +104,17 @@ def test_metadata_without_return_statement(self):
def test_get_class_metadata(self):
root = self.root_node

classes = list(PhpParser.get_class_list(root))[0]
metadata = PhpParser.get_class_metadata(classes)
_class, interface, trait = list(PhpParser.get_class_list(root))
class_metadata = PhpParser.get_class_metadata(_class)

self.assertEqual(metadata['parameters'], {'AbstractSQLServerDriver': None})
self.assertEqual(metadata['identifier'], 'Driver')
self.assertEqual(class_metadata['parameters'], {'AbstractSQLServerDriver': None})
self.assertEqual(class_metadata['identifier'], 'Driver')

interface_metadata = PhpParser.get_class_metadata(interface)
self.assertEqual(interface_metadata['identifier'], 'MyInterface')

trait_metadata = PhpParser.get_class_metadata(trait)
self.assertEqual(trait_metadata['identifier'], 'MyTrait')


if __name__ == '__main__':
Expand Down
27 changes: 23 additions & 4 deletions tests/test_parser/test_python.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ def test_get_class_list(self):
root = self.root_node

class_list = PythonParser.get_class_list(root)

self.assertEqual(len(class_list), 1)

def test_get_docstring(self):
Expand Down Expand Up @@ -59,21 +58,41 @@ def test_sample(arg1: str = "string", arg2 = "another_string"):

def test_get_class_metadata(self):
code_sample = '''
class ABC():
pass

class Sample(ABC):
def __init__(self):
pass

def test_sample(self, arg1: str = "string", arg2 = "another_string"):
return NotImplement()

class ThisIsalsoAclass(ABC, Sample):
pass
'''
root = parse_code(code_sample, 'python').root_node

classes = list(PythonParser.get_class_list(root))[0]
metadata = PythonParser.get_class_metadata(classes)


classes = list(PythonParser.get_class_list(root))
self.assertEqual(len(classes), 3)

metadata = PythonParser.get_class_metadata(classes[0])
self.assertEqual(metadata['parameters'], {})
self.assertEqual(metadata['identifier'], 'ABC')


metadata = PythonParser.get_class_metadata(classes[1])
self.assertEqual(metadata['parameters'], {'ABC': None})
self.assertEqual(metadata['identifier'], 'Sample')


metadata = PythonParser.get_class_metadata(classes[2])
self.assertEqual(metadata['parameters'], {'ABC': None, 'Sample': None})
self.assertEqual(metadata['identifier'], 'ThisIsalsoAclass')



def test_get_comment_list(self):
root = self.root_node

Expand Down
34 changes: 31 additions & 3 deletions tests/test_parser/test_ruby.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def test_get_function_list(self):

function_list = RubyParser.get_function_list(root)

self.assertEqual(len(function_list), 1)
self.assertEqual(len(function_list), 2)

def test_get_class_list(self):
root = self.root_node
Expand Down Expand Up @@ -76,15 +76,23 @@ def squeeze
def test_get_function_metadata(self):
root = self.root_node

function = RubyParser.get_function_list(root)[0]
metadata = RubyParser.get_function_metadata(function)
_function = RubyParser.get_function_list(root)[0]
metadata = RubyParser.get_function_metadata(_function)

for key in ['identifier', 'parameters', 'return_type']:
self.assertTrue(key in metadata.keys())
self.assertEqual(metadata['identifier'], 'search')
self.assertEqual(metadata['parameters'], {'query': None, 'options': None})
self.assertEqual(metadata['return_type'], None)

_singleton = RubyParser.get_function_list(root)[1]
metadata = RubyParser.get_function_metadata(_singleton)
for key in ['identifier', 'parameters', 'return_type']:
self.assertTrue(key in metadata.keys())
self.assertEqual(metadata['identifier'], 'my_method')
self.assertEqual(metadata['parameters'], {'a': None})
self.assertEqual(metadata['return_type'], '<not_specific>')


def test_metadata_without_return_statement(self):
code_sample = '''
Expand Down Expand Up @@ -114,6 +122,26 @@ def test_get_class_metadata(self):
self.assertEqual(metadata['identifier'], 'Client')
self.assertEqual(metadata['parameters'], {'API': None})

def test_get_action_list(self):
root = self.root_node
actions = RubyParser.get_action_list(root)

self.assertEqual(len(actions), 5)

def test_get_action_metadata(self):
root = self.root_node
actions = RubyParser.get_action_list(root)
metadatas = [ RubyParser.get_action_metadata(action) for action in actions]
self.assertEqual(metadatas[0]["identifier"], "load_current_value")
self.assertEqual(metadatas[1]["identifier"], "action:install")
self.assertEqual(metadatas[2]["identifier"], "converge_by")

self.assertEqual(metadatas[3]["identifier"], "action:reinstall")
self.assertEqual(metadatas[4]["identifier"], "converge_by")

self.assertEqual(metadatas[0]["parameters"]["new_resource"], None)
self.assertEqual(metadatas[0]["parameters"]["old_resource"], None)


if __name__ == '__main__':
unittest.main()
22 changes: 22 additions & 0 deletions tests/test_parser/test_sample/php_test_sample.php
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,25 @@ private function getConnectionOptionsDsn(array $connectionOptions): string
return $connectionOptionsDsn;
}
}

interface MyInterface {
public function myMethod() {
// Method implementation
}

}

trait MyTrait {

public function setBackgroundImage(Drawing $objDrawing): self
{
if (!array_key_exists($objDrawing->getType(), Drawing::IMAGE_TYPES_CONVERTION_MAP)) {
throw new PhpSpreadsheetException('Unsupported image type in comment background. Supported types: PNG, JPEG, BMP, GIF.');
}
$this->backgroundImage = $objDrawing;

return $this;
}

}

Loading