diff --git a/.gitignore b/.gitignore index 66ec4b2d1..1d2dce527 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ nbproject/ # Vim *.swp +# clangd cache +.cache/ ## Build folders build/ diff --git a/Config.cmake b/Config.cmake index 1032ce379..be9f13749 100644 --- a/Config.cmake +++ b/Config.cmake @@ -51,6 +51,9 @@ set(INSTALL_GEN_DIR "${INSTALL_SCRIPTS_DIR}/generated") # Installation directory for Java libraries set(INSTALL_JAVA_LIB_DIR "${INSTALL_LIB_DIR}/java") +# Installation directory for the Python plugin +set(INSTALL_PYTHON_DIR "${INSTALL_LIB_DIR}/pythonplugin") + # Installation directory for executables set(INSTALL_BIN_DIR "bin") diff --git a/doc/pythonplugin.md b/doc/pythonplugin.md new file mode 100644 index 000000000..f1e4442d0 --- /dev/null +++ b/doc/pythonplugin.md @@ -0,0 +1,123 @@ +# Python Plugin + +## Parsing Python projects +Python projects can be parsed by using the `CodeCompass_parser` executable. +See its usage [in a seperate document](/doc/usage.md). + +## Python specific parser flags + +### Python dependencies +Large Python projects usually have multiple Python package dependencies. +Although a given project can be parsed without installing any of its dependencies, it is strongly recommended +that the required modules are installed in order to achieve a complete parsing. +To install a project's dependencies, create a [Python virtual environment](https://docs.python.org/3/library/venv.html) +and install the necessary packages. +When parsing a project, specify the virtual environment path so the parser can successfully resolve the dependencies: +``` +--venvpath +``` + +### Type hints +The parser can try to determine Python type hints for variables, expressions and functions. +It can work out type hints such as `Iterable[int]` or `Union[int, str]`. +However, this process can be extremely slow, especially for functions, thus it is disabled by default. +It can be enabled using the `--type-hint` flag. + +### Python submodules +Large Python projects can have internal submodules and the parser tries to locate them automatically. +Specifically, it looks for `__init__.py` files and considers those folders modules. +This process is called submodule discovery and can be disabled using the `--disable-submodule-discovery` flag. + +You can also add submodules manually by adding those specific paths to the parser's syspath: +``` +--syspath +``` +For more information, see the [Python syspath docs](https://docs.python.org/3/library/sys.html#sys.path). + +### File references +By default, the parser works out references by looking for definitions only - if nodes share the same definition +they are considered references. +However, this method sometimes misses a few references (e.g. local variables in a function). +To extend search for references in a file context, apply the `--file-refs` flag. +Note that using this option can potentially extend the total parsing time. + +## Examples of parsing Python projects + +### Flask +We downloaded [flask 3.1.0](https://github.com/pallets/flask/releases/tag/3.1.0) source code to `~/parsing/flask/`. +The first step is to create a Python virtual environment and install flask's dependencies. +Create a Python virtual environment and activate it: +```bash +cd ~/parsing/flask/ +python3 -m venv venv +source venv/bin/activate +``` +Next, we install the required dependencies listed in `pyproject.toml`. +```bash +pip install . +``` +Further dependencies include development packages listed in `requirements/dev.txt`. +These can be also installed using `pip`. +```bash +pip install -r requirements/dev.txt +``` +Finally, we can run `CodeCompass_parser`. +```bash +CodeCompass_parser \ + -n flask \ + -i ~/parsing/flask/ \ + -w ~/parsing/workdir/ \ + -d "pgsql:host=localhost;port=5432;user=compass;password=pass;database=flask" \ + -f \ + --venvpath ~/parsing/flask/venv/ \ + --label src=~/parsing/flask/ +``` + +### CodeChecker +We downloaded [CodeChecker 6.24.4](https://github.com/Ericsson/codechecker/releases/tag/v6.24.4) source code to `~/parsing/codechecker`. +CodeChecker has an automated way of creating a Python virtual environment and installing dependencies - by running the `venv` target of a Makefile: +```bash +cd ~/parsing/codechecker/ +make venv +``` +Next, we can run `CodeCompass_parser`. +```bash +CodeCompass_parser \ + -n codechecker \ + -i ~/parsing/codechecker/ \ + -w ~/parsing/workdir/ \ + -d "pgsql:host=localhost;port=5432;user=compass;password=pass;database=codechecker" \ + -f \ + --venvpath ~/parsing/codechecker/venv/ \ + --label src=~/parsing/codechecker/ +``` + +## Troubleshooting +A few errors can occur during the parsing process, these are highlighted in color red. +The stack trace is hidden by default, and can be shown using the `--stack-trace` flag. + +### Failed to use virtual environment +This error can appear if one specifies the `--venvpath` option during parsing. +The parser tried to use the specified virtual environment path, however it failed. + +#### Solution +Double check that the Python virtual environment is correctly setup and its +path is correct. +If the error still persists, apply the `--stack-trace` parser option +to view a more detailed stack trace of the error. + +### Missing module (file = path line = number) +In this case, the parser tried to parse a given Python file, however it +could not find a definition for a module. +Commonly, the Python file imports another module and the parser cannot locate this module. +If this happens, the Python file is marked *partial* indicating that +a module definition was not resolved in this file. +The error message displays the module name, exact file path and line number +so one can further troubleshoot this problem. + +#### Solution +Ensure that the `--venvpath` option is correctly specified and all the required +dependencies are installed in that Python virtual environment. +If the imported module is part of the parsed project, use the `--syspath` option +and specify the directory where the module is located in. + diff --git a/plugins/python/CMakeLists.txt b/plugins/python/CMakeLists.txt new file mode 100644 index 000000000..ebd6783d8 --- /dev/null +++ b/plugins/python/CMakeLists.txt @@ -0,0 +1,6 @@ +add_subdirectory(model) +add_subdirectory(parser) +add_subdirectory(service) +add_subdirectory(test) + +install_webplugin(webgui) diff --git a/plugins/python/model/CMakeLists.txt b/plugins/python/model/CMakeLists.txt new file mode 100644 index 000000000..87e661e87 --- /dev/null +++ b/plugins/python/model/CMakeLists.txt @@ -0,0 +1,10 @@ +set(ODB_SOURCES + include/model/pyname.h +) + +generate_odb_files("${ODB_SOURCES}") + +add_odb_library(pythonmodel ${ODB_CXX_SOURCES}) +target_link_libraries(pythonmodel model) + +install_sql() diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h new file mode 100644 index 000000000..54f8e99dd --- /dev/null +++ b/plugins/python/model/include/model/pyname.h @@ -0,0 +1,53 @@ +#ifndef CC_MODEL_PYNAME_H +#define CC_MODEL_PYNAME_H + +#include +#include +#include + +namespace cc +{ +namespace model +{ + +enum PYNameID { + ID, + REF_ID, + PARENT, + PARENT_FUNCTION +}; + +#pragma db object +struct PYName +{ + #pragma db id unique + std::uint64_t id = 0; + + #pragma db index + std::uint64_t ref_id; + + std::uint64_t parent; + std::uint64_t parent_function; + + bool is_definition = false; + bool is_builtin = false; + bool is_import = false; + bool is_call = false; + std::string full_name; + std::string value; + std::string type; + std::string type_hint; + + std::uint64_t line_start; + std::uint64_t line_end; + std::uint64_t column_start; + std::uint64_t column_end; + + #pragma db index + std::uint64_t file_id; +}; + +} +} + +#endif diff --git a/plugins/python/parser/.gitignore b/plugins/python/parser/.gitignore new file mode 100644 index 000000000..f7275bbbd --- /dev/null +++ b/plugins/python/parser/.gitignore @@ -0,0 +1 @@ +venv/ diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt new file mode 100644 index 000000000..e0f7a1c54 --- /dev/null +++ b/plugins/python/parser/CMakeLists.txt @@ -0,0 +1,46 @@ +find_package(Python3 REQUIRED COMPONENTS Interpreter Development) +find_package(Boost REQUIRED COMPONENTS python) + +include_directories( + include + ${PROJECT_SOURCE_DIR}/model/include + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_SOURCE_DIR}/parser/include + ${PLUGIN_DIR}/model/include) + +include_directories(SYSTEM + ${Boost_INCLUDE_DIRS} + ${Python3_INCLUDE_DIRS}) + +add_library(pythonparser SHARED + src/pythonparser.cpp) + +target_link_libraries(pythonparser + model + pythonmodel + ${Boost_LIBRARIES} + ${Python3_LIBRARIES}) + +target_compile_options(pythonparser PUBLIC -Wno-unknown-pragmas) + +set(VENV_DIR "${PLUGIN_DIR}/parser/venv/") +if(NOT EXISTS ${VENV_DIR}) + message("Creating Python virtual environment: ${VENV_DIR}") + execute_process( + COMMAND python3 -m venv venv + WORKING_DIRECTORY ${PLUGIN_DIR}/parser/) +endif() + +message("Installing Python dependencies...") +execute_process( + COMMAND venv/bin/pip install -r requirements.txt + WORKING_DIRECTORY ${PLUGIN_DIR}/parser/) + +install(TARGETS pythonparser DESTINATION ${INSTALL_PARSER_DIR}) +install( + DIRECTORY pyparser/ + DESTINATION ${INSTALL_PYTHON_DIR}/pyparser) +install( + DIRECTORY venv/ + DESTINATION ${INSTALL_PYTHON_DIR}/venv) + diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h new file mode 100644 index 000000000..b3b1c25d8 --- /dev/null +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -0,0 +1,43 @@ +#ifndef CC_PARSER_PYTHONPARSER_H +#define CC_PARSER_PYTHONPARSER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace cc +{ +namespace parser +{ + +namespace python = boost::python; + +typedef std::unordered_map PYNameMap; + +class PythonParser : public AbstractParser +{ +public: + PythonParser(ParserContext& ctx_); + virtual ~PythonParser(); + virtual bool parse() override; +private: + struct ParseResultStats { + std::uint32_t partial; + std::uint32_t full; + }; + + python::object m_py_module; + void processFile(const python::object& obj, PYNameMap& map, ParseResultStats& parse_result); + void parseProject(const std::string& root_path); +}; + +} // parser +} // cc + +#endif // CC_PARSER_PYTHONPARSER_H diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py new file mode 100644 index 000000000..2fe495690 --- /dev/null +++ b/plugins/python/parser/pyparser/asthelper.py @@ -0,0 +1,255 @@ +import ast +from typing import cast, List +from posinfo import PosInfo +from nodeinfo import NodeInfo +from parserutil import fnvHash, getHashName +from parserconfig import ParserConfig + +class ASTHelper: + astNodes: List[ast.AST] + calls: List[ast.Call] + imports: List[ast.Import | ast.ImportFrom] + functions: List[ast.FunctionDef] + classes: List[ast.ClassDef] + path: str + source: str + lines: List[str] + + def __init__(self, path: str, source: str, config: ParserConfig): + self.astNodes = [] + self.calls = [] + self.imports = [] + self.functions = [] + self.classes = [] + self.path = path + self.file_id = fnvHash(self.path) + self.source = source + self.lines = source.split("\n") + self.config = config + + try: + tree = ast.parse(source) + self.astNodes = list(ast.walk(tree)) + except: + pass + + if config.ast: + if self.config.ast_function_call: + self.calls = self.__getFunctionCalls() + + if self.config.ast_import: + self.imports = self.__getImports() + + self.functions = self.__getFunctions() + self.classes = self.__getClasses() + + def __getFunctionCalls(self) -> List[ast.Call]: + return cast(List[ast.Call], list(filter(lambda e : isinstance(e, ast.Call), self.astNodes))) + + def __getFunctions(self) -> List[ast.FunctionDef]: + return cast(List[ast.FunctionDef], list(filter(lambda e : isinstance(e, ast.FunctionDef), self.astNodes))) + + def __getClasses(self) -> List[ast.ClassDef]: + return cast(List[ast.ClassDef], list(filter(lambda e : isinstance(e, ast.ClassDef), self.astNodes))) + + def __getImports(self) -> List[ast.Import | ast.ImportFrom]: + return cast(List[ast.Import | ast.ImportFrom], list(filter(lambda e : isinstance(e, ast.Import) or isinstance(e, ast.ImportFrom), self.astNodes))) + + def __equalPos(self, e: ast.expr | ast.stmt, pos: PosInfo): + return (e.lineno == pos.line_start and + e.end_lineno == pos.line_end and + e.col_offset == pos.column_start and + e.end_col_offset == pos.column_end) + + def isFunctionCall(self, pos: PosInfo): + for e in self.calls: + func = e.func + + if (isinstance(func, ast.Name)): + if (self.__equalPos(func, pos)): + return True + + elif (isinstance(func, ast.Attribute)): + if isinstance(func.end_col_offset, int): + col_start = func.end_col_offset - len(func.attr) + else: + col_start = 0 + + if (func.end_lineno == pos.line_start and + func.end_lineno == pos.line_end and + col_start == pos.column_start and + func.end_col_offset == pos.column_end): + return True + + return False + + def isImport(self, pos: PosInfo): + for e in self.imports: + if (self.__equalPos(e, pos)): + return True + + return False + + def getFunctionParam(self, pos: PosInfo) -> str | None: + for func in self.functions: + if isinstance(func.args, ast.arguments) and func.args.args: + for e in func.args.args: + if (e.lineno == pos.line_start and + e.col_offset == pos.column_start): + return self.__getPosValue(pos) + + return None + + def getSubclass(self, pos: PosInfo) -> int | None: + if not (self.config.ast_inheritance): + return None + + for cls in self.classes: + if not (isinstance(cls.lineno, int) and + isinstance(cls.end_lineno, int) and + isinstance(cls.col_offset, int)): + continue + + for e in cls.bases: + if (isinstance(e, ast.Name) and + e.lineno == pos.line_start and + e.end_lineno == pos.line_end and + e.col_offset == pos.column_start and + e.end_col_offset == pos.column_end): + + col_end = len(self.lines[cls.end_lineno - 1]) + subpos = PosInfo(line_start=cls.lineno, line_end=cls.end_lineno, column_start=cls.col_offset, column_end=col_end) + subhash = getHashName(self.path, subpos) + return subhash + + return None + + def __getPosValue(self, pos: PosInfo) -> str | None: + if not (pos.line_start >= 1 and pos.line_end >= 1): + return None + + if pos.line_start == pos.line_end: + value = self.lines[pos.line_start - 1][pos.column_start:pos.column_end] + else: + value = self.lines[pos.line_start - 1][pos.column_start:] + for l in range(pos.line_start, pos.line_end - 1): + value += self.lines[l] + value += self.lines[pos.line_end - 1][:pos.column_end] + + if value: + return value + else: + return None + + def __getASTValue(self, node: ast.expr) -> PosInfo | None: + line_end = node.end_lineno if node.end_lineno else node.lineno + col_end = node.end_col_offset if node.end_col_offset else node.col_offset + + pos = PosInfo(line_start=node.lineno, line_end=line_end, column_start=node.col_offset, column_end=col_end) + value = None + + if isinstance(node, ast.Subscript) or isinstance(node, ast.Attribute): + value = self.__getPosValue(pos) + elif isinstance(node, ast.Name): + value = node.id + elif isinstance(node, ast.Constant): + value = str(node.value) + + if value: + pos.value = value + return pos + else: + return None + + def __getFunctionReturnAnnotation(self, func: ast.FunctionDef) -> PosInfo | None: + if func.returns: + return self.__getASTValue(func.returns) + else: + return None + + def __getArgumentAnnotation(self, arg: ast.arg) -> PosInfo | None: + if arg.annotation: + return self.__getASTValue(arg.annotation) + else: + return None + + def __getFunctionSignature(self, func: ast.FunctionDef) -> str: + sign = "def " + func.name + sign += "(" + + first = True + for arg in func.args.args: + if first: + first = False + else: + sign += ", " + + sign += arg.arg + + param_annotation: PosInfo | None = self.__getArgumentAnnotation(arg) + if param_annotation: + sign += ": " + param_annotation.value + + sign += ")" + + return_annotation: PosInfo | None = self.__getFunctionReturnAnnotation(func) + if return_annotation: + sign += " -> " + return_annotation.value + + return sign + + def getAnnotations(self): + if not (self.config.ast and self.config.ast_annotations): + return [] + + results = [] + + for func in self.functions: + if not (isinstance(func.lineno, int) and + isinstance(func.end_lineno, int) and + isinstance(func.col_offset, int) and + isinstance(func.end_col_offset, int)): + continue + + subpos = self.__getFunctionReturnAnnotation(func) + + if subpos: + funcpos = PosInfo(line_start=func.lineno, line_end=func.end_lineno, column_start=func.col_offset, column_end=func.end_col_offset) + funchash = getHashName(self.path, funcpos) + subhash = getHashName(self.path, subpos) + + nodeinfo = NodeInfo( + id = subhash, + ref_id = subhash, + parent = funchash, + parent_function = funchash, + line_start = subpos.line_start, + line_end = subpos.line_end, + column_start = subpos.column_start, + column_end = subpos.column_end, + file_id = self.file_id, + type = "annotation", + value = subpos.value + ) + + results.append(nodeinfo) + + return results + + def getFunctionSignatureByPosition(self, pos: PosInfo) -> str | None: + if not (self.config.ast_function_signature): + return None + + for func in self.functions: + if not (isinstance(func.lineno, int) and + isinstance(func.end_lineno, int) and + isinstance(func.col_offset, int) and + isinstance(func.end_col_offset, int)): + continue + + if not (self.__equalPos(func, pos)): + continue + + return self.__getFunctionSignature(func) + + return None diff --git a/plugins/python/parser/pyparser/nodeinfo.py b/plugins/python/parser/pyparser/nodeinfo.py new file mode 100644 index 000000000..d97b6365d --- /dev/null +++ b/plugins/python/parser/pyparser/nodeinfo.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass + +@dataclass +class NodeInfo: + id: int = 0 + ref_id: int = 0 + file_id: int = 0 + + module_name: str = "" + full_name: str = "" + + parent: int = 0 + parent_function: int = 0 + + line_start: int = 1 + line_end: int = 1 + column_start: int = 1 + column_end: int = 1 + value: str = "" + + type: str = "" + type_hint: str = "" + is_definition: bool = False + is_builtin: bool = False + is_call: bool = False + is_import: bool = False diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py new file mode 100644 index 000000000..3201c3eb4 --- /dev/null +++ b/plugins/python/parser/pyparser/parser.py @@ -0,0 +1,136 @@ +import os +import jedi +import multiprocessing +import traceback +from itertools import repeat +from parserlog import log, bcolors, log_config +from asthelper import ASTHelper +from pyname import PYName +from parserconfig import ParserConfig +from nodeinfo import NodeInfo +from parseresult import ParseResult +from pyreference import PYReference +from pybuiltin import PYBuiltin + +def parseProject(settings, n_proc): + config = ParserConfig( + root_path=settings["root_path"], + venv_path=settings["venv_path"], + sys_path=settings["sys_path"], + debug=settings["debug"], + stack_trace=settings["stack_trace"], + type_hint=settings["type_hint"], + submodule_discovery=settings["submodule_discovery"], + ast=settings["ast"], + ast_function_call=settings["ast_function_call"], + ast_import=settings["ast_import"], + ast_annotations=settings["ast_annotations"], + ast_inheritance=settings["ast_inheritance"], + ast_function_signature=settings["ast_function_signature"], + file_refs=settings["file_refs"] + ) + + log(f"Parsing project: {config.root_path}") + log_config(config) + + py_files = [] + submodule_map = {} + for root, dirs, files in os.walk(config.root_path): + if config.venv_path and root.startswith(config.venv_path): + continue + + for file in files: + p = os.path.join(root, file) + ext = os.path.splitext(p)[1] + + if ext and ext.lower() == '.py': + py_files.append(p) + + if file == '__init__.py': + parent_dir = os.path.abspath(os.path.join(root, os.pardir)) + submodule_map[parent_dir] = True + + try: + if config.venv_path: + jedi.create_environment(config.venv_path, safe = config.safe_env) + log(f"{bcolors.OKGREEN}Using virtual environment: {config.venv_path}") + else: + config.venv_path = None + + if config.sys_path: + log(f"{bcolors.OKGREEN}Using additional syspath: {config.sys_path}") + + submodule_sys_path = list(submodule_map.keys()) + if config.submodule_discovery and submodule_sys_path: + log(f"{bcolors.OKBLUE}Submodule discovery results: {submodule_sys_path}") + config.sys_path.extend(submodule_sys_path) + + config.project = jedi.Project(path = config.root_path, environment_path = config.venv_path, added_sys_path = config.sys_path) + + except: + log(f"{bcolors.FAIL}Failed to use virtual environment: {config.venv_path}") + if config.stack_trace: + traceback.print_exc() + + log(f"{bcolors.OKGREEN}Using {n_proc} process to parse project") + + with multiprocessing.Pool(processes=n_proc) as pool: + results = pool.starmap(parse, zip(py_files, repeat(config))) + return results + +def parse(path: str, config: ParserConfig): + result: ParseResult = ParseResult(path=path) + + PYBuiltin.findBuiltins(config) + + nodes: dict[int, NodeInfo] = {} + imports: dict[str, bool] = {} + + with open(path) as f: + try: + log(f"Parsing: {path}") + source = f.read() + script = jedi.Script(source, path=path, project=config.project) + names = script.get_names(references = True, all_scopes = True) + + asthelper = ASTHelper(path, source, config) + pyref = PYReference(config, script, names) + + for e in asthelper.getAnnotations(): + putInMap(nodes, e) + + for x in names: + defs = pyref.getDefs(x) + refs = pyref.getFileRefs(x) + + putInMap(nodes, PYName(x).addConfig(config).addDefs(defs, result).addRefs(refs).addASTHelper(asthelper).getNodeInfo()) + + for d in defs: + if not (d.path): + continue + + # Builtin or library definition + if ((config.venv_path and d.path.startswith(config.venv_path)) or + not (d.path.startswith(config.root_path))): + putInMap(nodes, d.addConfig(config).getNodeInfo()) + imports[d.path] = True + + except: + log(f"{bcolors.FAIL}Failed to parse file: {path}") + if config.stack_trace: + traceback.print_exc() + + result.nodes = list(nodes.values()) + result.imports = list(imports.keys()) + + if len(result.nodes) == 0: + result.status = "none" + + return result + +def putInMap(hashmap: dict[int, NodeInfo], node: NodeInfo): + if node.id in hashmap: + return + + hashmap[node.id] = node + diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py new file mode 100644 index 000000000..1f59b7833 --- /dev/null +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import List +from jedi import Project + +@dataclass +class ParserConfig: + root_path: str + sys_path: List[str] + venv_path: str | None = None + project: Project | None = None + debug: bool = False + stack_trace: bool = False + safe_env: bool = False + type_hint: bool = False + file_refs: bool = False + submodule_discovery: bool = True + ast: bool = True + ast_function_call: bool = True + ast_import: bool = True + ast_annotations: bool = True + ast_inheritance: bool = True + ast_function_signature: bool = True diff --git a/plugins/python/parser/pyparser/parseresult.py b/plugins/python/parser/pyparser/parseresult.py new file mode 100644 index 000000000..cb5fbece6 --- /dev/null +++ b/plugins/python/parser/pyparser/parseresult.py @@ -0,0 +1,8 @@ +from dataclasses import dataclass + +@dataclass +class ParseResult: + path: str + status: str = "full" + nodes = [] + imports = [] diff --git a/plugins/python/parser/pyparser/parserlog.py b/plugins/python/parser/pyparser/parserlog.py new file mode 100644 index 000000000..1c0dcc73a --- /dev/null +++ b/plugins/python/parser/pyparser/parserlog.py @@ -0,0 +1,55 @@ +import datetime +from parserconfig import ParserConfig + +def log(msg): + date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print(f"{date} [pythonparser] {msg}{bcolors.ENDC}") + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +def log_config(config: ParserConfig): + if config.debug: + log(f"{bcolors.WARNING}Parsing in debug mode!") + + if config.type_hint: + log(f"{bcolors.OKGREEN}Type hint support enabled!") + else: + log(f"{bcolors.OKBLUE}Type hint support disabled!") + + if config.file_refs: + log(f"{bcolors.OKGREEN}File refs lookup enabled!") + else: + log(f"{bcolors.OKBLUE}File refs lookup disabled!") + + if config.stack_trace: + log(f"{bcolors.WARNING}Stack trace enabled!") + + if not (config.submodule_discovery): + log(f"{bcolors.WARNING}Submodule discovery disabled!") + + if not (config.ast): + log(f"{bcolors.WARNING}All AST modules disabled!") + + if not (config.ast_function_call): + log(f"{bcolors.WARNING}AST function call parsing disabled!") + + if not (config.ast_import): + log(f"{bcolors.WARNING}AST import parsing disabled!") + + if not (config.ast_inheritance): + log(f"{bcolors.WARNING}AST inheritance parsing disabled!") + + if not (config.ast_annotations): + log(f"{bcolors.WARNING}AST annotation parsing disabled!") + + if not (config.ast_function_signature): + log(f"{bcolors.WARNING}AST function signature parsing disabled!") diff --git a/plugins/python/parser/pyparser/parserutil.py b/plugins/python/parser/pyparser/parserutil.py new file mode 100644 index 000000000..15707e64a --- /dev/null +++ b/plugins/python/parser/pyparser/parserutil.py @@ -0,0 +1,17 @@ +from hashlib import sha1 +from posinfo import PosInfo + +def getHashName(path, pos: PosInfo) -> int: + s = f"{path}|{pos.line_start}|{pos.line_end}|{pos.column_start}|{pos.column_end}".encode("utf-8") + hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff + return hash + +def fnvHash(str) -> int: + hash = 14695981039346656037 + + for c in str: + hash ^= ord(c) + hash *= 1099511628211 + + # see: https://stackoverflow.com/questions/20766813/how-to-convert-signed-to-unsigned-integer-in-python + return hash & 0xffffffffffffffff diff --git a/plugins/python/parser/pyparser/posinfo.py b/plugins/python/parser/pyparser/posinfo.py new file mode 100644 index 000000000..7a22021f0 --- /dev/null +++ b/plugins/python/parser/pyparser/posinfo.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + +@dataclass +class PosInfo: + line_start: int = 0 + line_end: int = 0 + column_start: int = 0 + column_end: int = 0 + value: str = "" + diff --git a/plugins/python/parser/pyparser/pybuiltin.py b/plugins/python/parser/pyparser/pybuiltin.py new file mode 100644 index 000000000..c16f7cae8 --- /dev/null +++ b/plugins/python/parser/pyparser/pybuiltin.py @@ -0,0 +1,44 @@ +import sys +import os +import importlib.util +import traceback +from jedi.api.classes import Name +from parserlog import log, bcolors +from parserconfig import ParserConfig + +class PYBuiltin: + builtin = {} + + @staticmethod + def findBuiltins(config: ParserConfig): + try: + # Note: Python 3.10+ required + stdlib_modules = sys.stdlib_module_names + + for e in stdlib_modules: + spec = importlib.util.find_spec(e) + if spec and spec.origin: + PYBuiltin.builtin[spec.origin] = True + + if spec and spec.submodule_search_locations: + for submodule_dir in spec.submodule_search_locations: + for root, dirs, files in os.walk(submodule_dir): + for file in files: + p = os.path.join(root, file) + ext = os.path.splitext(p)[1] + + if ext and ext.lower() == '.py': + PYBuiltin.builtin[p] = True + + except: + log(f"{bcolors.FAIL}Failed to find Python builtins!") + if config.stack_trace: + traceback.print_exc() + + @staticmethod + def isBuiltin(name: Name): + path = str(name.module_path) + return (path in PYBuiltin.builtin or + name.in_builtin_module() or + "/typeshed/stdlib/" in path) + diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py new file mode 100644 index 000000000..c12862a43 --- /dev/null +++ b/plugins/python/parser/pyparser/pyname.py @@ -0,0 +1,166 @@ +from jedi.api.classes import Name +from typing import List +from parserutil import fnvHash, getHashName +from parserlog import log, bcolors +from asthelper import ASTHelper +from posinfo import PosInfo +from pybuiltin import PYBuiltin +from parserconfig import ParserConfig +from nodeinfo import NodeInfo +from parseresult import ParseResult + +class PYName: + id: int + path: str | None + name: Name + pos: PosInfo + refid: int + defs: List['PYName'] + asthelper: ASTHelper | None + config: ParserConfig | None + + def __init__(self, name: Name): + self.name = name + self.path = str(self.name.module_path) if self.name.module_path else None + self.pos = self.__getNamePosInfo() + self.hashName = getHashName(self.path, self.pos) + self.refid = self.hashName + self.defs = [] + self.asthelper = None + self.config = None + + def addDefs(self, defs: List['PYName'], result): + self.defs = defs + + if len(defs) > 0: + self.refid = min(list(map(lambda e : e.hashName, defs))) + else: + self.__reportMissingDefinition(result) + + return self + + def addRefs(self, refs: List['PYName']): + defs = list(filter(lambda e : e.name.is_definition(), refs)) + + if len(defs) > 0: + self.refid = min(list(map(lambda e : e.hashName, defs))) + + return self + + def addASTHelper(self, asthelper: ASTHelper): + if self.config and self.config.ast: + self.asthelper = asthelper + + return self + + def addConfig(self, config: ParserConfig): + self.config = config + return self + + def __getNamePosInfo(self) -> PosInfo: + pos = PosInfo() + + start_pos = self.name.get_definition_start_position() + end_pos = self.name.get_definition_end_position() + + if start_pos and end_pos: + pos.line_start = start_pos[0] + pos.line_end = end_pos[0] + pos.column_start = start_pos[1] + pos.column_end = end_pos[1] + + if pos.line_start == pos.line_end: + pos.value = self.name.get_line_code()[pos.column_start:pos.column_end] + else: + pos.value = self.name.get_line_code()[pos.column_start:] + + if (self.path and + pos.line_start == 0 and pos.line_end == 0 and + pos.column_start == 0 and pos.column_end == 0): + pos.value = self.path.split("/")[-1] + + return pos + + def getNodeInfo(self) -> NodeInfo: + node = NodeInfo() + node.id = self.hashName + node.ref_id = self.refid + node.module_name = self.name.module_name + node.full_name = self.name.full_name if self.name.full_name else "" + + node.line_start = self.pos.line_start + node.line_end = self.pos.line_end + node.column_start = self.pos.column_start + 1 + node.column_end = self.pos.column_end + 1 + node.value = self.pos.value + + node.type = self.name.type + node.is_definition = self.name.is_definition() + node.file_id = self.__getFileId() + node.type_hint = self.__getNameTypeHint() + node.is_builtin = PYBuiltin.isBuiltin(self.name) or any(list(map(lambda x : PYBuiltin.isBuiltin(x.name), self.defs))) + + parent = self.name.parent() + node.parent = PYName(parent).hashName if parent else node.id + node.parent_function = self.__getParentFunction() + + if self.asthelper: + node.is_call = self.asthelper.isFunctionCall(self.pos) + node.is_import = self.asthelper.isImport(self.pos) + + if node.type == "param": + param = self.asthelper.getFunctionParam(self.pos) + if param: + node.type = "astparam" + node.value = param + + if node.type == "function" and node.is_definition and not (node.is_import or node.is_builtin): + sign = self.asthelper.getFunctionSignatureByPosition(self.pos) + node.value = sign if sign else node.value + + subclass = self.asthelper.getSubclass(self.pos) + if subclass: + node.type = "baseclass" + node.parent = subclass + + return node + + def __getFileId(self): + if self.path: + return fnvHash(self.path) + else: + return 0 + + def __reportMissingDefinition(self, result: ParseResult): + if not self.name.is_definition() and self.name.type == 'module': + log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.path} line = {self.pos.line_start})") + result.status = "partial" + + def __getParentFunction(self): + try: + node = self.name + for _ in range(0,10): + parent: Name | None = node.parent() + if parent and parent.type == "function" and parent.is_definition(): + return PYName(parent).hashName + elif parent: + node = parent + else: + break + except: + pass + + return self.hashName + + def __getNameTypeHint(self): + hint = "" + if not (self.config and self.config.type_hint): + return hint + + try: + res = self.name.get_type_hint() + hint = res if res else "" + except: + pass + + return hint diff --git a/plugins/python/parser/pyparser/pyreference.py b/plugins/python/parser/pyparser/pyreference.py new file mode 100644 index 000000000..70f2ff289 --- /dev/null +++ b/plugins/python/parser/pyparser/pyreference.py @@ -0,0 +1,61 @@ +from typing import List +from jedi import Script +from jedi.api.classes import Name +from pyname import PYName +from parserconfig import ParserConfig +from parserlog import log, bcolors +import traceback + +class PYReference: + script: Script + names: List[Name] + config: ParserConfig + + def __init__(self, config: ParserConfig, script: Script, names: List[Name]): + self.config = config + self.script = script + self.names = names + self.refmap = {} + + if self.config.file_refs: + self.__lookupFileRefs() + + def getDefs(self, name: Name) -> List[PYName]: + try: + defs = name.goto(follow_imports = True, follow_builtin_imports = True) + defs = list(map(lambda e : PYName(e), defs)) + return defs + except: + if self.config.debug: + log(f"{bcolors.FAIL}Failed to find definition! (file = {str(name.module_path)} line = {name.line} column = {name.column})") + if self.config.stack_trace: + traceback.print_exc() + + return [] + + def getFileRefs(self, name: Name) -> List[PYName]: + id = PYName(name).hashName + + if id in self.refmap: + return self.refmap[id] + else: + return [] + + def __lookupFileRefs(self): + for x in self.names: + if not(x.is_definition()): + continue + + try: + refs = self.script.get_references(x.line, x.column, scope = "file") + refs = list(map(lambda e : PYName(e), refs)) + refs.append(PYName(x)) + + for r in refs: + self.refmap[r.hashName] = refs + except: + if self.config.debug: + log(f"{bcolors.FAIL}Failed to find references! (file = {str(x.module_path)} line = {x.line} column = {x.column})") + if self.config.stack_trace: + traceback.print_exc() + diff --git a/plugins/python/parser/requirements.txt b/plugins/python/parser/requirements.txt new file mode 100644 index 000000000..18d880e9d --- /dev/null +++ b/plugins/python/parser/requirements.txt @@ -0,0 +1,2 @@ +jedi==0.18.0 +parso==0.8.4 diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp new file mode 100644 index 000000000..f52c8703b --- /dev/null +++ b/plugins/python/parser/src/pythonparser.cpp @@ -0,0 +1,235 @@ +#include +#include +#include + +namespace cc +{ +namespace parser +{ + +PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) +{ + // Init Python Interpreter + std::string py_parser_dir = _ctx.compassRoot + "/lib/pythonplugin/pyparser"; + std::string py_venv_dir = _ctx.compassRoot + "/lib/pythonplugin/venv"; + std::string path_env = py_venv_dir + "/bin" + ":" + getenv("PATH"); + + // Set Python module path + setenv("PYTHONPATH", py_parser_dir.c_str(), 1); + + // Activate Python venv + setenv("VIRTUAL_ENV", py_venv_dir.c_str(), 1); + setenv("PATH", path_env.c_str(), 1); + + Py_Initialize(); + + // Init PyParser module + try { + m_py_module = python::import("parser"); + } + catch (const python::error_already_set&) + { + PyErr_Print(); + } +} + +void PythonParser::parseProject(const std::string& root_path) +{ + PYNameMap map; + ParseResultStats parse_result; + parse_result.full = 0; + parse_result.partial = 0; + + try { + python::list sys_path; + int n_proc = _ctx.options["jobs"].as(); + + if(_ctx.options.count("syspath")) + { + std::vector vec = _ctx.options["syspath"].as>(); + for(const std::string& s : vec) + { + sys_path.append(s); + } + } + + python::dict settings; + settings["root_path"] = root_path; + settings["sys_path"] = sys_path; + settings["venv_path"] = (_ctx.options.count("venvpath")) ? _ctx.options["venvpath"].as() : ""; + settings["debug"] = (bool)(_ctx.options.count("debug")); + settings["stack_trace"] = (bool)(_ctx.options.count("stack-trace")); + settings["type_hint"] = (bool)(_ctx.options.count("type-hint")); + settings["submodule_discovery"] = !(_ctx.options.count("disable-submodule-discovery")); + settings["ast"] = !(_ctx.options.count("disable-ast")); + settings["ast_function_call"] = !(_ctx.options.count("disable-ast-function-call")); + settings["ast_import"] = !(_ctx.options.count("disable-ast-import")); + settings["ast_annotations"] = !(_ctx.options.count("disable-ast-annotations")); + settings["ast_inheritance"] = !(_ctx.options.count("disable-ast-inheritance")); + settings["ast_function_signature"] = !(_ctx.options.count("disable-ast-function-signature")); + settings["file_refs"] = (bool)(_ctx.options.count("file-refs")); + + python::object result_list = m_py_module.attr("parseProject")(settings, n_proc); + + for(int i = 0; i < python::len(result_list); i++) + { + PythonParser::processFile(result_list[i], map, parse_result); + } + + + }catch (const python::error_already_set&) + { + PyErr_Print(); + } + + // Insert into database + LOG(info) << "[pythonparser] Inserting PYNames to database..."; + cc::util::OdbTransaction {_ctx.db} ([&] + { + for(const auto& e : map) + { + _ctx.db->persist(e.second); + } + }); + + LOG(info) << "[pythonparser] Parsing finished!"; + LOG(info) << "[pythonparser] Inserted rows: " << map.size(); + LOG(info) << "[pythonparser] Fully parsed files: " << parse_result.full; + LOG(info) << "[pythonparser] Partially parsed files: " << parse_result.partial; +} + +void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseResultStats& parse_result) +{ + try { + python::object nodes = obj.attr("nodes"); + const std::string status = python::extract(obj.attr("status")); + const std::string path = python::extract(obj.attr("path")); + + const int len = python::len(nodes); + for (int i = 0; i < len; i++) + { + python::object node = nodes[i]; + + model::PYName pyname; + pyname.id = python::extract(node.attr("id")); + pyname.ref_id = python::extract(node.attr("ref_id")); + pyname.parent = python::extract(node.attr("parent")); + pyname.parent_function = python::extract(node.attr("parent_function")); + pyname.full_name = python::extract(node.attr("full_name")); + pyname.is_definition = python::extract(node.attr("is_definition")); + pyname.is_import = python::extract(node.attr("is_import")); + pyname.is_builtin = python::extract(node.attr("is_builtin")); + pyname.line_start = python::extract(node.attr("line_start")); + pyname.line_end = python::extract(node.attr("line_end")); + pyname.column_start = python::extract(node.attr("column_start")); + pyname.column_end = python::extract(node.attr("column_end")); + pyname.file_id = python::extract(node.attr("file_id")); + pyname.value = python::extract(node.attr("value")); + pyname.type = python::extract(node.attr("type")); + pyname.type_hint = python::extract(node.attr("type_hint")); + pyname.is_call = python::extract(node.attr("is_call")); + + // Put in map + if(map.find(pyname.id) == map.end()) + { + map[pyname.id] = pyname; + } + } + + if(status != "none") + { + model::FilePtr pyfile = _ctx.srcMgr.getFile(path); + + if(status == "full") + { + parse_result.full++; + pyfile->parseStatus = model::File::ParseStatus::PSFullyParsed; + }else if (status == "partial") + { + parse_result.partial++; + pyfile->parseStatus = model::File::ParseStatus::PSPartiallyParsed; + } + + pyfile->type = "PY"; + _ctx.srcMgr.updateFile(*pyfile); + } + + // Additional paths (example: builtin definition paths) + // These files need to be added to db + python::object imports = obj.attr("imports"); + for (int i = 0; i < python::len(imports); i++) + { + std::string p = python::extract(imports[i]); + + model::FilePtr file = _ctx.srcMgr.getFile(p); + file->type = "PY"; + _ctx.srcMgr.updateFile(*file); + } + + }catch (const python::error_already_set&) + { + PyErr_Print(); + } +} + +bool PythonParser::parse() +{ + for(const std::string& path : _ctx.options["input"].as>()) + { + PythonParser::parseProject(path); + } + + return true; +} + +PythonParser::~PythonParser() +{ +} + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ + boost::program_options::options_description getOptions() + { + boost::program_options::options_description description("Python Plugin"); + description.add_options() + ("venvpath", po::value(), + "Set 'venvpath' to specify the project's Python virtual environment path.") + ("syspath", po::value>(), + "Additional sys path for the parser.") + ("stack-trace", + "Enable error stack trace.") + ("type-hint", + "Enable type hint parsing.") + ("disable-submodule-discovery", + "Disable submodule disovery. Submodules will not be added to sys path.") + ("disable-ast", + "Disable all AST parsing modules.") + ("disable-ast-function-call", + "Disable AST function call parsing.") + ("disable-ast-import", + "Disable AST import parsing.") + ("disable-ast-annotations", + "Disable AST annotation parsing.") + ("disable-ast-inheritance", + "Disable AST inheritance parsing.") + ("disable-ast-function-signature", + "Disable AST function signature parsing.") + ("file-refs", + "Enable search for references in a file context.") + ("debug", + "Enable parsing in debug mode."); + + return description; + } + + std::shared_ptr make(ParserContext& ctx_) + { + return std::make_shared(ctx_); + } +} +#pragma clang diagnostic pop + +} // parser +} // cc diff --git a/plugins/python/service/CMakeLists.txt b/plugins/python/service/CMakeLists.txt new file mode 100644 index 000000000..17b63f5f2 --- /dev/null +++ b/plugins/python/service/CMakeLists.txt @@ -0,0 +1,31 @@ +include_directories( + include + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_SOURCE_DIR}/webserver/include + ${PROJECT_BINARY_DIR}/service/language/gen-cpp + ${PROJECT_BINARY_DIR}/service/project/gen-cpp + ${PROJECT_SOURCE_DIR}/service/project/include + ${PLUGIN_DIR}/model/include) + +include_directories(SYSTEM + ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}) + +add_library(pythonservice SHARED + src/pythonservice.cpp + src/diagram.cpp + src/plugin.cpp) + +target_compile_options(pythonservice PUBLIC -Wno-unknown-pragmas) + +target_link_libraries(pythonservice + util + model + pythonmodel + mongoose + projectservice + languagethrift + ${THRIFT_LIBTHRIFT_LIBRARIES} + ${ODB_LIBRARIES}) + +install(TARGETS pythonservice DESTINATION ${INSTALL_SERVICE_DIR}) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h new file mode 100644 index 000000000..efbb7aca0 --- /dev/null +++ b/plugins/python/service/include/service/pythonservice.h @@ -0,0 +1,175 @@ +#ifndef CC_SERVICE_PYTHON_PYTHONSERVICE_H +#define CC_SERVICE_PYTHON_PYTHONSERVICE_H + +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include + +#include +#include +#include + +namespace cc +{ +namespace service +{ +namespace language +{ + +class PythonServiceHandler : virtual public LanguageServiceIf +{ + friend class PythonDiagram; +public: + PythonServiceHandler( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_); + + void getFileTypes(std::vector& return_) override; + + void getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) override; + + void getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) override; + + void getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) override; + + void getDocumentation( + std::string&, + const core::AstNodeId&) override {}; + + void getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) override; + + void getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getFileDiagramTypes( + std::map& return_, + const core::FileId& fileId_) override; + + void getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) override; + + void getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) override; + + void getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector& tags_) override; + + std::int32_t getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) override; + + void getReferencesInFile( + std::vector&, + const core::AstNodeId&, + const std::int32_t, + const core::FileId&, + const std::vector&) override {}; + + void getReferencesPage( + std::vector&, + const core::AstNodeId&, + const std::int32_t, + const std::int32_t, + const std::int32_t) override {}; + + void getFileReferenceTypes( + std::map&, + const core::FileId&) override {}; + + void getFileReferences( + std::vector&, + const core::FileId&, + const std::int32_t) override {}; + + std::int32_t getFileReferenceCount( + const core::FileId&, + const std::int32_t) override { return 0; }; + + void getSyntaxHighlight( + std::vector&, + const core::FileRange&) override {}; + + enum ReferenceType + { + DEFINITION, + USAGE, + THIS_CALLS, + CALLER, + PARAMETER, + LOCAL_VAR, + DATA_MEMBER, + METHOD, + PARENT, + PARENT_FUNCTION, + ANNOTATION, + BASE_CLASS + }; + + enum DiagramType + { + FUNCTION_CALL, + MODULE_DEPENDENCY, + FUNCTION_USAGE, + CLASS_USAGE, + CLASS_OVERVIEW + }; + +private: + std::shared_ptr _db; + util::OdbTransaction _transaction; + std::shared_ptr _datadir; + const cc::webserver::ServerContext& _context; + + void setInfoProperties(AstNodeInfo& info, const model::PYName& pyname); + model::PYName queryNodeByID(const std::string& id); + model::PYName queryNodeByPosition(const core::FilePosition& fpos); + std::vector queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); + std::vector queryNodesInFile(const core::FileId& fileId, bool definitions); + std::vector queryNodes(const odb::query& odb_query); + std::vector transformReferences(const std::vector& references, const model::PYNameID& id); + std::string getNodeLineValue(const model::PYName& pyname); +}; + +} // language +} // service +} // cc + +#endif // CC_SERVICE_PYTHON_PYTHONSERVICE_H diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp new file mode 100644 index 000000000..36a978b36 --- /dev/null +++ b/plugins/python/service/src/diagram.cpp @@ -0,0 +1,522 @@ +#include "diagram.h" + +namespace cc +{ +namespace service +{ +namespace language +{ +PythonDiagram::PythonDiagram( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_) + : m_pythonService(db_, datadir_, context_), + m_projectService(db_, datadir_, context_){} + +util::Graph PythonDiagram::getFunctionCallDiagram(const model::PYName& pyname) +{ + // Query calls + const std::vector this_calls = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::THIS_CALLS); + std::vector this_calls_refs = m_pythonService.transformReferences(this_calls, model::REF_ID); + const std::vector calls = m_pythonService.queryNodes(odb::query::ref_id.in_range(this_calls_refs.begin(), this_calls_refs.end()) && odb::query::is_definition == true && odb::query::is_import == false); + + // Query callers + const std::vector function_callers = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::CALLER); + std::vector callers_parents = m_pythonService.transformReferences(function_callers, model::PARENT_FUNCTION); + const std::vector callers = m_pythonService.queryNodes(odb::query::id.in_range(callers_parents.begin(), callers_parents.end())); + + // Create graph + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + // Add center node + util::Graph::Node centerNode = addPYNameNode(graph, pyname, true); + const std::string centerNodeID = std::to_string(pyname.id); + decorateNode(graph, centerNode, CenterNode); + + // Add calls with definitions + std::unordered_map addedNodes; + for (const model::PYName& node : calls) + { + addFunctionNode(graph, centerNode, node, FunctionCallDefinitionNode); + addedNodes.emplace(node.ref_id, true); + } + + // Add calls with missing definitions + for(const model::PYName& node : this_calls) + { + if(addedNodes.find(node.ref_id) == addedNodes.end()) + { + addFunctionNode(graph, centerNode, node, FunctionCallNode); + } + } + + addedNodes.clear(); + + // Add callers with definitions + for (const model::PYName& node : callers) + { + addFunctionNode(graph, centerNode, node, FunctionCallerDefinitionNode); + } + + // Add callers with missing definitions + for (const model::PYName& node : function_callers) + { + if(node.parent_function == node.id) + { + addFunctionNode(graph, centerNode, node, FunctionCallerNode); + } + } + + return graph; +} + +util::Graph PythonDiagram::getModuleDiagram(const core::FileId& fileId) +{ + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + const std::uint64_t file_id = std::stoull(fileId); + core::FileInfo mainFileInfo; + m_projectService.getFileInfo(mainFileInfo, fileId); + util::Graph::Node centerNode = addFileNode(graph, mainFileInfo); + decorateNode(graph, centerNode, FilePathCenterNode); + + // Query nodes + const std::vector nodesInFile = m_pythonService.transformReferences(m_pythonService.queryNodesInFile(fileId, false), model::REF_ID); + const std::vector definitionsInFile = m_pythonService.transformReferences(m_pythonService.queryNodesInFile(fileId, true), model::REF_ID); + + const std::vector importedDefinitions = m_pythonService.queryNodes( + odb::query::ref_id.in_range(nodesInFile.begin(), nodesInFile.end()) && odb::query::is_definition == true && + odb::query::is_import == false && odb::query::file_id != file_id && + !(odb::query::line_start == 0 && odb::query::line_end == 0 && odb::query::column_start == 1 && odb::query::column_end == 1)); + + const std::vector importedUsages = m_pythonService.queryNodes( + odb::query::ref_id.in_range(definitionsInFile.begin(), definitionsInFile.end()) && odb::query::is_definition == false && + odb::query::file_id != file_id); + + std::unordered_map map; + + auto getFileNode = [&](const model::PYName& p, const NodeType& nodeType) + { + core::FileInfo fileInfo; + try { + m_projectService.getFileInfo(fileInfo, std::to_string(p.file_id)); + } catch (const core::InvalidId&) { + return util::Graph::Node(); + } + + auto it = map.find(p.file_id); + + if (it == map.end()) + { + util::Graph::Node n = addFileNode(graph, fileInfo, centerNode, nodeType); + + map.emplace(p.file_id, n); + return n; + }else{ + return it->second; + } + }; + + for (const model::PYName& p : importedDefinitions) + { + util::Graph::Node node = getFileNode(p, p.is_builtin ? ImportedBuiltinFilePathNode : ImportedFilePathNode); + if (node.empty()) continue; + + util::Graph::Node graphNode = addPYNameNode(graph, p, false); + decorateNode(graph, graphNode, ImportedNode); + graph.createEdge(node, graphNode); + } + + map.clear(); + + for (const model::PYName& p : importedUsages) + { + util::Graph::Node node = getFileNode(p, ImportsFilePathNode); + } + return graph; +} + +util::Graph PythonDiagram::getUsageDiagram(const model::PYName& pyname) +{ + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + util::Graph::Node centerNode = addPYNameNode(graph, pyname, true); + decorateNode(graph, centerNode, CenterNode); + + if (!pyname.is_definition || pyname.is_import == true) return graph; + + const std::vector usages = m_pythonService.queryNodes( + odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false); + + for (const model::PYName& p : usages) + { + util::Graph::Node graphNode = addPYNameNode(graph, p, true); + decorateNode(graph, graphNode, p.is_call ? FunctionCallNode : UsageNode); + graph.createEdge(graphNode, centerNode); + } + return graph; +} + +void PythonDiagram::addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType) +{ + util::Graph::Node node = addPYNameNode(graph_, pyname, true); + decorateNode(graph_, node, nodeType); + + if(nodeType == FunctionCallNode || nodeType == FunctionCallDefinitionNode) + { + graph_.createEdge(centerNode, node); + } + else + { + graph_.createEdge(node, centerNode); + } +} + +util::Graph::Subgraph PythonDiagram::getFileSubgraph(util::Graph& graph_, const model::PYName& pyname) +{ + const core::FileId fileId = std::to_string(pyname.file_id); + auto it = m_subgraphs.find(fileId); + + if (it != m_subgraphs.end()) { + return it->second; + } + + core::FileInfo fileInfo; + try { + m_projectService.getFileInfo(fileInfo, fileId); + } catch (const core::InvalidId&) { + return util::Graph::Subgraph(); + } + + util::Graph::Subgraph subgraph = graph_.getOrCreateSubgraph("cluster_" + fileInfo.path); + graph_.setSubgraphAttribute(subgraph, "id", fileInfo.id); + + const std::string pathColor = (pyname.is_builtin && pyname.is_definition) ? "dodgerblue" : "limegreen"; + const std::string coloredLabel = + "
" + + getRelativePath(fileInfo.path) + + "
"; + graph_.setSubgraphAttribute(subgraph, "label", coloredLabel, true); + + m_subgraphs.emplace(fileInfo.path, subgraph); + + return subgraph; +} + +util::Graph::Node PythonDiagram::addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph) +{ + const util::Graph::Subgraph subgraph = (addSubgraph) ? getFileSubgraph(graph_, pyname) : util::Graph::Subgraph(); + + util::Graph::Node node = graph_.getOrCreateNode(std::to_string(pyname.id), subgraph); + + std::string label = pyname.value; + + if(!pyname.is_definition) + { + label = m_pythonService.getNodeLineValue(pyname); + } + + graph_.setNodeAttribute(node, "label", label); + + return node; +} + +util::Graph::Node PythonDiagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo) +{ + util::Graph::Node node = graph_.getOrCreateNode("f" + fileInfo.id); + const std::string path = getRelativePath(fileInfo.path); + graph_.setNodeAttribute(node, "label", path); + + return node; +} + +util::Graph::Node PythonDiagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType) +{ + /* We might need to add a file path multiple times to the diagram. + Since we need unique ids, we will differentiate nodes based on starting character: + 'f' - regular file path node + 'd' - ImportedFilePathNode, ImportedBuiltinFilePathNode + 's' - ImportsFilePathNode + Any id without a {'f', 'd', 's'} starting character is not a file path node. */ + const std::string id = (nodeType == ImportedFilePathNode || nodeType == ImportedBuiltinFilePathNode) ? "d" + fileInfo.id : "s" + fileInfo.id; + + util::Graph::Node node = graph_.getOrCreateNode(id); + const std::string path = getRelativePath(fileInfo.path); + graph_.setNodeAttribute(node, "label", path); + decorateNode(graph_, node, nodeType); + + if (nodeType == ImportedFilePathNode || nodeType == ImportedBuiltinFilePathNode) { + graph_.createEdge(centerNode, node); + } else { + graph_.createEdge(node, centerNode); + } + + return node; +} + +void PythonDiagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeType& nodeType) +{ + graph_.setNodeAttribute(node_, "style", "filled"); + + switch(nodeType) + { + case CenterNode: + graph_.setNodeAttribute(node_, "fillcolor", "gold"); + break; + case FunctionCallerNode: + graph_.setNodeAttribute(node_, "fillcolor", "orange"); + graph_.setNodeAttribute(node_, "shape", "cds"); + break; + case FunctionCallerDefinitionNode: + graph_.setNodeAttribute(node_, "fillcolor", "coral"); + break; + case FunctionCallNode: + graph_.setNodeAttribute(node_, "fillcolor", "deepskyblue"); + graph_.setNodeAttribute(node_, "shape", "cds"); + break; + case FunctionCallDefinitionNode: + graph_.setNodeAttribute(node_, "fillcolor", "lightblue"); + break; + case FilePathCenterNode: + graph_.setNodeAttribute(node_, "fillcolor", "gold"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; + case ImportedFilePathNode: + graph_.setNodeAttribute(node_, "fillcolor", "limegreen"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; + case ImportedBuiltinFilePathNode: + graph_.setNodeAttribute(node_, "fillcolor", "dodgerblue"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; + case ImportedNode: + graph_.setNodeAttribute(node_, "fillcolor", "lightseagreen"); + break; + case ImportsFilePathNode: + graph_.setNodeAttribute(node_, "fillcolor", "orange"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; + case UsageNode: + graph_.setNodeAttribute(node_, "fillcolor", "cyan"); + graph_.setNodeAttribute(node_, "shape", "cds"); + break; + } +} + +util::Graph PythonDiagram::getFunctionCallDiagramLegend() +{ + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + addLegendNode(graph, ImportedBuiltinFilePathNode, "Builtin file path", false); + addLegendNode(graph, ImportedFilePathNode, "File path", false); + addLegendNode(graph, FunctionCallNode, "Function call without definition"); + addLegendNode(graph, FunctionCallDefinitionNode, "Function call"); + addLegendNode(graph, FunctionCallerNode, "Line of code calling selected node"); + addLegendNode(graph, FunctionCallerDefinitionNode, "Function calling selected node"); + addLegendNode(graph, CenterNode, "Selected node"); + + return graph; +} + +util::Graph PythonDiagram::getUsageDiagramLegend() +{ + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + addLegendNode(graph, ImportedBuiltinFilePathNode, "Builtin module path", false); + addLegendNode(graph, ImportedFilePathNode, "Module path", false); + addLegendNode(graph, FunctionCallNode, "Function call"); + addLegendNode(graph, UsageNode, "Usage node"); + addLegendNode(graph, CenterNode, "Selected node"); + + return graph; +} + +util::Graph PythonDiagram::getModuleDiagramLegend() +{ + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + addLegendNode(graph, ImportedNode, "Imported definition"); + addLegendNode(graph, ImportedBuiltinFilePathNode, "Imported builtin module"); + addLegendNode(graph, ImportedFilePathNode, "Imported module"); + addLegendNode(graph, ImportsFilePathNode, "Module importing selected node"); + addLegendNode(graph, FilePathCenterNode, "Selected node"); + + return graph; +} + +void PythonDiagram::addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text, bool shape) +{ + util::Graph::Node node = graph_.createNode(); + graph_.setNodeAttribute(node, "label", ""); + decorateNode(graph_, node, nodeType); + + if (!shape) { + graph_.setNodeAttribute(node, "shape", "plaintext"); + } + + const util::Graph::Node explanation = graph_.createNode(); + graph_.setNodeAttribute(explanation, "shape", "none"); + + const util::Graph::Edge edge = graph_.createEdge(node, explanation); + graph_.setEdgeAttribute(edge, "style", "invis"); + + graph_.setNodeAttribute(explanation, "label", text); +} + +util::Graph PythonDiagram::getClassDiagram(const model::PYName& pyname) +{ + util::Graph graph; + graph.setAttribute("rankdir", "BT"); + + auto setAttributes = [this, &graph](const util::Graph::Node& node, const model::PYName& p) { + graph.setNodeAttribute(node, "fontname", "Noto Serif"); + graph.setNodeAttribute(node, "shape", "plaintext"); + graph.setNodeAttribute(node, "label", getClassTable(p), true); + }; + + util::Graph::Node classNode = graph.getOrCreateNode(std::to_string(pyname.id), getFileSubgraph(graph, pyname)); + setAttributes(classNode, pyname); + + // Query baseclasses + const std::vector bases = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::BASE_CLASS); + + for (const model::PYName& p : bases) { + if (p.type != "class") { + continue; + } + + util::Graph::Node node = graph.getOrCreateNode(std::to_string(p.id), getFileSubgraph(graph, p)); + setAttributes(node, p); + + util::Graph::Edge edge = graph.createEdge(classNode, node); + graph.setEdgeAttribute(edge, "arrowhead", "empty"); + } + + return graph; +} + +std::string PythonDiagram::getClassTable(const model::PYName& pyname) +{ + auto getVisibility = [](const std::string& str) + { + if (str.substr(0, 6) == "def __") { + return std::string("- "); + } else { + return std::string("+ "); + } + }; + + auto highlightVariable = [](const std::string& str, const std::string& baseColor) + { + // Remove comma + std::string p = (str.back() == ',') ? str.substr(0, str.size() - 1) : str; + + const size_t col = p.find(":"); + const size_t eq = p.find("="); + const size_t hashtag = p.find("#"); + + if (col != std::string::npos && eq != std::string::npos && col < eq && hashtag == std::string::npos) { + p = "" + p.substr(0, col) + "" + ":" + + "" + p.substr(col + 1, eq - col - 1) + "" + + "" + p.substr(eq) + ""; + } else if (col != std::string::npos && eq == std::string::npos && hashtag == std::string::npos) { + p = "" + p.substr(0, col) + "" + ":" + + "" + p.substr(col + 1) + ""; + } else { + p = "" + p + ""; + } + + return p; + }; + + auto getSignature = [this, &highlightVariable](const model::PYName& p) + { + const size_t opening = p.value.find('('); + if (p.value.substr(0, 3) != "def" || opening == std::string::npos) { + return highlightVariable(p.value, "black"); + } + + // Remove "def" + std::string sign = p.value.substr(3, opening - 3); + sign += '('; + + // Query params + const std::vector params = m_pythonService.queryReferences(std::to_string(p.id), PythonServiceHandler::PARAMETER); + + // Add params to signature + bool first = true; + for (const model::PYName& e : params) { + // Skip param "self" + if (first && e.value.substr(0,4) == "self") { + continue; + } + + if (first) { + first = false; + } else { + sign += ", "; + } + + sign += highlightVariable(e.value, "darkgreen"); + } + + sign += ')'; + + // Query return annotation + const std::vector annotations = m_pythonService.queryReferences(std::to_string(p.id), PythonServiceHandler::ANNOTATION); + if(annotations.size() == 1) { + sign += " -> " + annotations[0].value + ""; + } + + return sign; + }; + + // Query nodes + const std::vector data_members = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::DATA_MEMBER); + const std::vector methods = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::METHOD); + + std::string label = ""; + + label += ""; + + label += ""; + label += "
" + pyname.value + "
"; + for (const model::PYName& p : data_members) { + label += "
" + getVisibility(p.value) + getSignature(p) + "
"; + } + label += "
"; + for (const model::PYName& p : methods) { + label += "
" + getVisibility(p.value) + getSignature(p) + "
"; + } + label += "
"; + return label; +} + +std::string PythonDiagram::getRelativePath(const std::string& path) +{ + std::map labels; + m_projectService.getLabels(labels); + + if (labels.count("src")) { + std::string projectPath = labels["src"]; + std::string projectPathWithSlash = labels["src"] + "/"; + + if (path.substr(0, projectPathWithSlash.size()) == projectPathWithSlash) { + return path.substr(projectPathWithSlash.size()); + } + + if (path.substr(0, projectPath.size()) == projectPath) { + return path.substr(projectPath.size()); + } + } + + return path; +} +} // language +} // service +} // cc diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h new file mode 100644 index 000000000..6a1e1385b --- /dev/null +++ b/plugins/python/service/src/diagram.h @@ -0,0 +1,66 @@ +#ifndef CC_SERVICE_PYTHON_DIAGRAM_H +#define CC_SERVICE_PYTHON_DIAGRAM_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cc +{ +namespace service +{ +namespace language +{ + class PythonDiagram { + public: + PythonDiagram( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_); + + util::Graph getFunctionCallDiagram(const model::PYName& pyname); + util::Graph getModuleDiagram(const core::FileId& fileId); + util::Graph getUsageDiagram(const model::PYName& pyname); + util::Graph getClassDiagram(const model::PYName& pyname); + util::Graph getFunctionCallDiagramLegend(); + util::Graph getUsageDiagramLegend(); + util::Graph getModuleDiagramLegend(); + + enum NodeType { + CenterNode, + FunctionCallNode, + FunctionCallDefinitionNode, + FunctionCallerNode, + FunctionCallerDefinitionNode, + FilePathCenterNode, + ImportedFilePathNode, + ImportedBuiltinFilePathNode, + ImportsFilePathNode, + ImportedNode, + UsageNode + }; + private: + PythonServiceHandler m_pythonService; + core::ProjectServiceHandler m_projectService; + std::map m_subgraphs; + void addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType); + void addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text, bool shape = true); + void decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeType& nodeType); + util::Graph::Node addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph); + util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType); + util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo); + util::Graph::Subgraph getFileSubgraph(util::Graph& graph_, const model::PYName& pyname); + std::string getClassTable(const model::PYName& pyname); + std::string getRelativePath(const std::string& path); + }; +} // language +} // service +} // cc + +#endif diff --git a/plugins/python/service/src/plugin.cpp b/plugins/python/service/src/plugin.cpp new file mode 100644 index 000000000..424c8f947 --- /dev/null +++ b/plugins/python/service/src/plugin.cpp @@ -0,0 +1,28 @@ +#include + +#include + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ + boost::program_options::options_description getOptions() + { + namespace po = boost::program_options; + + po::options_description description("Python Plugin"); + return description; + } + + void registerPlugin( + const cc::webserver::ServerContext& context_, + cc::webserver::PluginHandler* pluginHandler_) + { + cc::webserver::registerPluginSimple( + context_, + pluginHandler_, + CODECOMPASS_LANGUAGE_SERVICE_FACTORY_WITH_CFG(Python), + "PythonService"); + } +} +#pragma clang diagnostic pop diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp new file mode 100644 index 000000000..0d28e4bf8 --- /dev/null +++ b/plugins/python/service/src/pythonservice.cpp @@ -0,0 +1,534 @@ +#include +#include +#include +#include +#include +#include "diagram.h" + +namespace cc +{ +namespace service +{ +namespace language +{ + +PythonServiceHandler::PythonServiceHandler( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_) + : _db(db_), _transaction(db_), _datadir(datadir_), _context(context_) {} + +void PythonServiceHandler::getFileTypes( + std::vector& return_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + return_.push_back("PY"); + return_.push_back("Dir"); + return; +} + +void PythonServiceHandler::getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); + + PythonServiceHandler::setInfoProperties(return_, pyname); + return; +} + +void PythonServiceHandler::getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + model::PYName node = PythonServiceHandler::queryNodeByPosition(fpos_); + PythonServiceHandler::setInfoProperties(return_, node); + return; +} + +void PythonServiceHandler::getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); + + core::ProjectServiceHandler projectService(_db, _datadir, _context); + std::string content; + projectService.getFileContent(content, std::to_string(pyname.file_id)); + + if(!content.empty()) + { + return_ = cc::util::textRange( + content, + pyname.line_start, + pyname.column_start, + pyname.line_end, + pyname.column_end); + } + return; +} + +void PythonServiceHandler::getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); + + if(!pyname.full_name.empty()) + { + return_.emplace("Full name", pyname.full_name); + } + + return_.emplace("Builtin", util::boolToString(pyname.is_builtin)); + + if(!pyname.type_hint.empty()) + { + return_.emplace("Type hint", pyname.type_hint); + } + + return_.emplace("Function call", util::boolToString(pyname.is_call)); + +#ifndef NDEBUG + return_.emplace("ID", std::to_string(pyname.id)); + return_.emplace("REF_ID", std::to_string(pyname.ref_id)); + return_.emplace("DEFINITION", util::boolToString(pyname.is_definition)); +#endif + + return; +} + +void PythonServiceHandler::getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + const model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); + + if (pyname.is_import == true) return; + + if (pyname.is_definition == true && pyname.type == "function") { + return_.emplace("Function call", FUNCTION_CALL); + } + + if (pyname.is_definition == true && pyname.type == "class") { + return_.emplace("Class Overview", CLASS_OVERVIEW); + } + + // Usage diagrams + const size_t count = PythonServiceHandler::queryReferences(astNodeId_, USAGE).size(); + + if (count > 0 && pyname.is_definition == true) { + if (pyname.type == "function") { + return_.emplace("Function usage", FUNCTION_USAGE); + } else if (pyname.type == "class") { + return_.emplace("Class Usage", CLASS_USAGE); + } + } + + return; +} + +void PythonServiceHandler::getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + PythonDiagram diagram(_db, _datadir, _context); + + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); + util::Graph graph = [&]() + { + switch (diagramId_) + { + case FUNCTION_CALL: + return diagram.getFunctionCallDiagram(pyname); + case FUNCTION_USAGE: + case CLASS_USAGE: + return diagram.getUsageDiagram(pyname); + case CLASS_OVERVIEW: + return diagram.getClassDiagram(pyname); + default: + return util::Graph(); + } + }(); + + if (graph.nodeCount() != 0) { + if (diagramId_ == CLASS_OVERVIEW) { + return_ = graph.output(util::Graph::CAIRO_SVG); + } else { + return_ = graph.output(util::Graph::SVG); + } + } + + return; +} + +void PythonServiceHandler::getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + PythonDiagram diagram(_db, _datadir, _context); + + util::Graph graph = [&]() + { + switch (diagramId_) + { + case FUNCTION_CALL: + return diagram.getFunctionCallDiagramLegend(); + case FUNCTION_USAGE: + case CLASS_USAGE: + return diagram.getUsageDiagramLegend(); + default: + return util::Graph(); + } + }(); + + if (graph.nodeCount() != 0) + return_ = graph.output(util::Graph::SVG); + + return; +} + +void PythonServiceHandler::getFileDiagramTypes( + std::map& return_, + const core::FileId&) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + return_.emplace("Module dependency", MODULE_DEPENDENCY); + return; +} + +void PythonServiceHandler::getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + PythonDiagram diagram(_db, _datadir, _context); + + util::Graph graph = [&]() + { + switch (diagramId_) + { + case MODULE_DEPENDENCY: + return diagram.getModuleDiagram(fileId_); + default: + return util::Graph(); + } + }(); + + if (graph.nodeCount() != 0) + return_ = graph.output(util::Graph::SVG); + + return; +} + +void PythonServiceHandler::getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + PythonDiagram diagram(_db, _datadir, _context); + + util::Graph graph = [&]() + { + switch (diagramId_) + { + case MODULE_DEPENDENCY: + return diagram.getModuleDiagramLegend(); + default: + return util::Graph(); + } + }(); + + if (graph.nodeCount() != 0) + return_ = graph.output(util::Graph::SVG); + + return; +} + +void PythonServiceHandler::getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif + return_.emplace("Definition", DEFINITION); + return_.emplace("Usage", USAGE); + return_.emplace("Parent", PARENT); + return_.emplace("Parameters", PARAMETER); + return_.emplace("Caller", CALLER); + + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId); + + if(pyname.type == "function" && pyname.is_definition) + { + return_.emplace("Method", METHOD); + return_.emplace("Local variables", LOCAL_VAR); + return_.emplace("This calls", THIS_CALLS); + } + + std::vector nodes = PythonServiceHandler::queryReferences(astNodeId, DEFINITION); + + if(!nodes.empty()) + { + model::PYName def = *nodes.begin(); + if(def.type == "class") + { + return_.emplace("Method", METHOD); + return_.emplace("Data member", DATA_MEMBER); + return_.emplace("Base class", BASE_CLASS); + } + } + + return; +} + +void PythonServiceHandler::getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector&) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; + LOG(info) << "astNodeID: " << astNodeId_; +#endif + std::vector nodes = PythonServiceHandler::queryReferences(astNodeId_, referenceId_); + + for(const model::PYName& pyname : nodes) + { + AstNodeInfo info; + PythonServiceHandler::setInfoProperties(info, pyname); + return_.push_back(info); + } + + return; +} + +std::int32_t PythonServiceHandler::getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) +{ +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE] " << __func__; + LOG(info) << "astNodeID: " << astNodeId_; +#endif + + return PythonServiceHandler::queryReferences(astNodeId_, referenceId_).size(); +} + +model::PYName PythonServiceHandler::queryNodeByID(const std::string& id) +{ + return _transaction([&]() + { + model::PYName pyname; + odb::result nodes = _db->query(odb::query::id == std::stoull(id)); + + if(!nodes.empty()) + { + pyname = *nodes.begin(); + }else{ + LOG(info) << "[PYTHONSERVICE] Node not found! (id = " << id << ")"; + core::InvalidId ex; + ex.__set_msg("Node not found!"); + throw ex; + } + + return pyname; + }); +} + +model::PYName PythonServiceHandler::queryNodeByPosition(const core::FilePosition& fpos) +{ + return _transaction([&]() + { + model::PYName pyname; + odb::result nodes = _db->query( + odb::query::file_id == std::stoull(fpos.file) && + odb::query::line_start == fpos.pos.line && + odb::query::column_start <= fpos.pos.column && + odb::query::column_end >= fpos.pos.column + ); + + if(!nodes.empty()) + { + pyname = *nodes.begin(); + + for(const model::PYName& p : nodes) + { + if(p.value.size() < pyname.value.size()) + { + pyname = p; + } + } + }else{ + LOG(info) << "[PYTHONSERVICE] Node not found! (line = " << fpos.pos.line << " column = " << fpos.pos.column << ")"; + core::InvalidInput ex; + ex.__set_msg("Node not found!"); + throw ex; + } + + return pyname; + }); +} + +std::vector PythonServiceHandler::queryNodes(const odb::query& odb_query) +{ + return _transaction([&](){ + const odb::query order_by = "ORDER BY" + odb::query::is_builtin + "," + odb::query::line_start + "," + odb::query::column_start; + const odb::query q = odb_query + order_by; + + odb::result nodes = _db->query(q); + + return std::vector(nodes.begin(), nodes.end()); + }); +} + +std::vector PythonServiceHandler::queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId) +{ + return _transaction([&](){ + odb::result nodes; + + const model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId); + const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; + + switch (referenceId) + { + case DEFINITION: + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false) + order_by); + break; + case USAGE: + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::id != pyname.id) + order_by); + break; + case METHOD: + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "function" && odb::query::is_definition == true) + order_by); + break; + case LOCAL_VAR: + case DATA_MEMBER: + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "statement" && odb::query::is_definition == true) + order_by); + break; + case PARENT: + nodes = _db->query((odb::query::id == pyname.parent) + order_by); + break; + case PARENT_FUNCTION: + nodes = _db->query((odb::query::id == pyname.parent_function) + order_by); + break; + case PARAMETER: + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "astparam" && odb::query::is_definition == true) + order_by); + break; + case CALLER: + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::is_call == true && odb::query::id != pyname.id) + order_by); + break; + case THIS_CALLS: + nodes = _db->query((odb::query::parent == pyname.id && odb::query::is_call == true) + order_by); + break; + case ANNOTATION: + nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "annotation") + order_by); + break; + case BASE_CLASS: + odb::result bases = _db->query((odb::query::parent == pyname.id && odb::query::type == "baseclass")); + const std::vector bases_refs = PythonServiceHandler::transformReferences(std::vector(bases.begin(), bases.end()), model::REF_ID); + + nodes = _db->query((odb::query::ref_id.in_range(bases_refs.begin(), bases_refs.end()) && odb::query::is_definition == true && odb::query::is_import == false) + order_by); + break; + } + + return std::vector(nodes.begin(), nodes.end()); + }); +} + +std::vector PythonServiceHandler::queryNodesInFile(const core::FileId& fileId, bool definitions) +{ + return _transaction([&](){ + const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; + odb::result nodes = (definitions) ? + _db->query((odb::query::file_id == std::stoull(fileId) && odb::query::is_definition == true && odb::query::is_import == false) + order_by) + : _db->query((odb::query::file_id == std::stoull(fileId)) + order_by); + + return std::vector(nodes.begin(), nodes.end()); + }); +} + +std::vector PythonServiceHandler::transformReferences(const std::vector& references, const model::PYNameID& id) +{ + std::vector ret(references.size()); + switch(id) + { + case model::ID: + std::transform(references.begin(), references.end(), ret.begin(), [](const model::PYName& e){return e.id;}); + break; + case model::REF_ID: + std::transform(references.begin(), references.end(), ret.begin(), [](const model::PYName& e){return e.ref_id;}); + break; + case model::PARENT: + std::transform(references.begin(), references.end(), ret.begin(), [](const model::PYName& e){return e.parent;}); + break; + case model::PARENT_FUNCTION: + std::transform(references.begin(), references.end(), ret.begin(), [](const model::PYName& e){return e.parent_function;}); + break; + } + return ret; +} + +void PythonServiceHandler::setInfoProperties(AstNodeInfo& info, const model::PYName& pyname) +{ + info.id = std::to_string(pyname.id); + info.astNodeValue = pyname.value; + info.symbolType = pyname.type; + info.range.file = std::to_string(pyname.file_id); + info.range.range.startpos.line = pyname.line_start; + info.range.range.startpos.column = pyname.column_start; + info.range.range.endpos.line = pyname.line_end; + info.range.range.endpos.column = pyname.column_end; + info.astNodeType = pyname.type_hint; +} + +std::string PythonServiceHandler::getNodeLineValue(const model::PYName& pyname) +{ + core::ProjectServiceHandler projectService(_db, _datadir, _context); + std::string content; + projectService.getFileContent(content, std::to_string(pyname.file_id)); + + std::istringstream iss(content); + std::string lineStr; + + for (std::size_t i = 1; i <= pyname.line_start; ++i) + { + std::getline(iss, lineStr); + } + + return lineStr; +} +} // language +} // service +} // cc diff --git a/plugins/python/test/CMakeLists.txt b/plugins/python/test/CMakeLists.txt new file mode 100644 index 000000000..773771795 --- /dev/null +++ b/plugins/python/test/CMakeLists.txt @@ -0,0 +1,68 @@ +if (NOT FUNCTIONAL_TESTING_ENABLED) + fancy_message("Skipping generation of test project pythontest." "yellow" TRUE) + return() +endif() + +fancy_message("Generating test project for pythontest." "blue" TRUE) + +include_directories( + ${PLUGIN_DIR}/model/include + ${PLUGIN_DIR}/service/include + ${PROJECT_SOURCE_DIR}/model/include + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_BINARY_DIR}/service/language/gen-cpp + ${PROJECT_BINARY_DIR}/service/project/gen-cpp) + +add_executable(pythonparsertest + src/pythontest.cpp + src/pythonparsertest.cpp) + +add_executable(pythonservicetest + src/pythontest.cpp + src/pythonservicetest.cpp) + +target_compile_options(pythonparsertest PUBLIC -Wno-unknown-pragmas) +target_compile_options(pythonservicetest PUBLIC -Wno-unknown-pragmas) + +target_link_libraries(pythonparsertest + ${GTEST_BOTH_LIBRARIES} + pthread + util + model + pythonmodel) + +target_link_libraries(pythonservicetest + ${GTEST_BOTH_LIBRARIES} + pthread + util + model + pythonmodel + pythonservice) + +# Clean up the build folder and the output of the test in a `make clean`. +set_property(DIRECTORY APPEND PROPERTY + ADDITIONAL_MAKE_CLEAN_FILES + "${CMAKE_CURRENT_BINARY_DIR}/workdir") + +# CTest runs the `pythonparsertest` binary with two arguments: +# 1. Shell script which parses the Python project +# 2. Test database connection string +add_test(NAME pythonparser COMMAND pythonparsertest + "echo \"Test database used: ${TEST_DB}\" && \ + ${CMAKE_INSTALL_PREFIX}/bin/CodeCompass_parser \ + --database \"${TEST_DB}\" \ + --name pythonparsertest \ + --input ${CMAKE_CURRENT_SOURCE_DIR}/sources/ \ + --workspace ${CMAKE_CURRENT_BINARY_DIR}/workdir/ \ + --force" + "${TEST_DB}") + +add_test(NAME pythonservice COMMAND pythonservicetest + "echo \"Test database used: ${TEST_DB}\" && \ + ${CMAKE_INSTALL_PREFIX}/bin/CodeCompass_parser \ + --database \"${TEST_DB}\" \ + --name pythonservicetest \ + --input ${CMAKE_CURRENT_SOURCE_DIR}/sources/ \ + --workspace ${CMAKE_CURRENT_BINARY_DIR}/workdir/ \ + --force" + "${TEST_DB}") diff --git a/plugins/python/test/sources/.gitignore b/plugins/python/test/sources/.gitignore new file mode 100644 index 000000000..bee8a64b7 --- /dev/null +++ b/plugins/python/test/sources/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/plugins/python/test/sources/classes.py b/plugins/python/test/sources/classes.py new file mode 100644 index 000000000..888315f33 --- /dev/null +++ b/plugins/python/test/sources/classes.py @@ -0,0 +1,31 @@ +class Base: + DEBUG_MODE = False + users = [] + + def __init__(self) -> None: + pass + + def foo(self): + pass + + def bar(self): + print("bar") + + def test(): + pass + + test() + +class Derived(Base): + def __init__(self) -> None: + pass + +class Derived2(Derived, Base): + def __init__(self) -> None: + pass + +base = Base() + +def func(): + class A: + z = 2 diff --git a/plugins/python/test/sources/functions.py b/plugins/python/test/sources/functions.py new file mode 100644 index 000000000..5fe172291 --- /dev/null +++ b/plugins/python/test/sources/functions.py @@ -0,0 +1,99 @@ +def hello_world(): + print("Hello, world!") + +def runner(func, param1, param2): + return func(param1,param2) + +def mul(a, b): + return a * b + +def mul2(a, b): + return mul(a,b) + +def mul3(): + return mul + +mylib = { + "multiply": mul +} + +class MyLib: + def __init__(self): + self.multiply = mul + +mylib2 = MyLib() + +a = mul(4,8) +a2 = runner(mul,4,8) +a3 = mul3()(4,8) +a4 = mylib["multiply"](4,8) +a5 = mylib2.multiply(4,8) + +if __name__ == "__main__": + hello_world() + + print(a) + print(a2) + print(a3) + print(a4) + print(a5) + + print("----------") + + print(mul(4,8)) + print(runner(mul,4,8)) + print(mul3()(4,8)) + print(mylib["multiply"](4,8)) + print(mylib2.multiply(4,8)) + +def sign(a: int, b: str) -> None: + pass + +def sign2( + a: int, + b: str) -> None: + pass + +def sign3( + a: int = 2, + b: str = "hi") -> None: + pass + +def sign4( # note + a: int = 2, # note 2 + b: str = "hi") -> None: # note 3 + pass + +from typing import List, Optional + +def annotation(a, b) -> None: + pass + +def annotation2(a, + b) -> str: + return "abc" + +def annotation3(a, + b) -> int: + return 0 + +def annotation4(a, + b) -> bool: + return True + +def annotation5(a, + b) -> List[str]: + return [] + +def annotation6(a, + b) -> Optional[str]: + pass + +def annotation7(a, + b) -> dict[int, bool]: + return {} + +def local_var(): + a = 2 + for i in range(0,10): + a += i diff --git a/plugins/python/test/sources/imports.py b/plugins/python/test/sources/imports.py new file mode 100644 index 000000000..82b538afb --- /dev/null +++ b/plugins/python/test/sources/imports.py @@ -0,0 +1,12 @@ +import classes +import os +from functions import mul + +a = mul(4,8) +print(a) +print(mul(4,8)) + +base = classes.Base() +base.bar() + +print("pid", os.getpid()) diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp new file mode 100644 index 000000000..c656e4b29 --- /dev/null +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -0,0 +1,489 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace cc; +extern const char* dbConnectionString; + +class PythonParserTest : public ::testing::Test +{ +public: + PythonParserTest() : + _db(util::connectDatabase(dbConnectionString)), + _transaction(_db) + { + loadFile("functions.py"); + loadFile("classes.py"); + loadFile("imports.py"); + } + + model::PYName queryFile(const std::string& filename, const odb::query& odb_query) + { + model::PYName pyname; + if (m_files.count(filename)) + { + _transaction([&, this]() { + pyname = _db->query_value(odb_query && odb::query::file_id == m_files[filename]); + }); + } + + return pyname; + } + +private: + std::unordered_map m_files; + void loadFile(const std::string& filename) + { + _transaction([&, this]() { + model::File file = _db->query_value(odb::query::filename == filename); + m_files.emplace(filename, file.id); + }); + } + +protected: + std::shared_ptr _db; + util::OdbTransaction _transaction; +}; + +TEST_F(PythonParserTest, FilesAreInDatabase) +{ + _transaction([&, this]() { + model::File file; + + file = _db->query_value(odb::query::filename == "functions.py"); + EXPECT_EQ(file.type, "PY"); + EXPECT_EQ(file.parseStatus, model::File::PSFullyParsed); + + file = _db->query_value(odb::query::filename == "classes.py"); + EXPECT_EQ(file.type, "PY"); + EXPECT_EQ(file.parseStatus, model::File::PSFullyParsed); + + file = _db->query_value(odb::query::filename == "imports.py"); + EXPECT_EQ(file.type, "PY"); + EXPECT_EQ(file.parseStatus, model::File::PSFullyParsed); + }); +} + +TEST_F(PythonParserTest, FunctionDefinition) +{ + model::PYName pyname; + + pyname = queryFile("functions.py", odb::query::value == "def hello_world()"); + EXPECT_EQ(pyname.is_definition, true); + + pyname = queryFile("functions.py", odb::query::value == "def runner(func, param1, param2)"); + EXPECT_EQ(pyname.is_definition, true); + + pyname = queryFile("functions.py", odb::query::value == "def mul(a, b)"); + EXPECT_EQ(pyname.is_definition, true); + + pyname = queryFile("functions.py", odb::query::value == "def mul2(a, b)"); + EXPECT_EQ(pyname.is_definition, true); + + pyname = queryFile("functions.py", odb::query::value == "def mul3()"); + EXPECT_EQ(pyname.is_definition, true); +} + +TEST_F(PythonParserTest, FunctionType) +{ + model::PYName pyname; + + pyname = queryFile("functions.py", odb::query::value == "def hello_world()"); + EXPECT_EQ(pyname.type, "function"); + + pyname = queryFile("functions.py", odb::query::value == "def runner(func, param1, param2)"); + EXPECT_EQ(pyname.type, "function"); + + pyname = queryFile("functions.py", odb::query::value == "def mul(a, b)"); + EXPECT_EQ(pyname.type, "function"); + + pyname = queryFile("functions.py", odb::query::value == "def mul2(a, b)"); + EXPECT_EQ(pyname.type, "function"); + + pyname = queryFile("functions.py", odb::query::value == "def mul3()"); + EXPECT_EQ(pyname.type, "function"); +} + +TEST_F(PythonParserTest, FunctionParamAST) +{ + model::PYName pyname; + + pyname = queryFile("functions.py", odb::query::line_start == 4 && + odb::query::column_start == 12); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = queryFile("functions.py", odb::query::line_start == 4 && + odb::query::column_start == 18); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = queryFile("functions.py", odb::query::line_start == 4 && + odb::query::column_start == 26); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = queryFile("functions.py", odb::query::line_start == 7 && + odb::query::column_start == 9); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = queryFile("functions.py", odb::query::line_start == 7 && + odb::query::column_start == 12); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = queryFile("functions.py", odb::query::line_start == 10 && + odb::query::column_start == 10); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = queryFile("functions.py", odb::query::line_start == 10 && + odb::query::column_start == 13); + EXPECT_EQ(pyname.type, "astparam"); +} + +TEST_F(PythonParserTest, FunctionSignatureAST) +{ + model::PYName pyname; + + pyname = queryFile("functions.py", odb::query::line_start == 49 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign(a: int, b: str) -> None"); + + pyname = queryFile("functions.py", odb::query::line_start == 52 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign2(a: int, b: str) -> None"); + + pyname = queryFile("functions.py", odb::query::line_start == 57 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign3(a: int, b: str) -> None"); + + pyname = queryFile("functions.py", odb::query::line_start == 62 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign4(a: int, b: str) -> None"); +} + +TEST_F(PythonParserTest, FunctionAnnotationAST) +{ + model::PYName pyname; + + pyname = queryFile("functions.py", odb::query::line_start == 69 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "None"); + + pyname = queryFile("functions.py", odb::query::line_start == 73 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "str"); + + pyname = queryFile("functions.py", odb::query::line_start == 77 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "int"); + + pyname = queryFile("functions.py", odb::query::line_start == 81 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "bool"); + + pyname = queryFile("functions.py", odb::query::line_start == 85 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "List[str]"); + + pyname = queryFile("functions.py", odb::query::line_start == 89 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "Optional[str]"); + + pyname = queryFile("functions.py", odb::query::line_start == 93 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "dict[int, bool]"); +} + +TEST_F(PythonParserTest, FunctionCall) +{ + model::PYName pyname; + + pyname = queryFile("functions.py", odb::query::value == "def hello_world()"); + EXPECT_EQ(pyname.is_call, false); + + pyname = queryFile("functions.py", odb::query::value == "def runner(func, param1, param2)"); + EXPECT_EQ(pyname.is_call, false); + + pyname = queryFile("functions.py", odb::query::value == "def mul(a, b)"); + EXPECT_EQ(pyname.is_call, false); + + pyname = queryFile("functions.py", odb::query::value == "def mul2(a, b)"); + EXPECT_EQ(pyname.is_call, false); + + pyname = queryFile("functions.py", odb::query::value == "def mul3()"); + EXPECT_EQ(pyname.is_call, false); + + // ---------- + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 11); + EXPECT_EQ(pyname.is_call, true); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 14); + EXPECT_EQ(pyname.is_call, false); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 17); + EXPECT_EQ(pyname.is_call, false); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 22); + EXPECT_EQ(pyname.is_call, false); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 26); + EXPECT_EQ(pyname.is_call, true); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 27); + EXPECT_EQ(pyname.is_call, false); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 43); + EXPECT_EQ(pyname.is_call, true); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 44); + EXPECT_EQ(pyname.is_call, false); +} + +TEST_F(PythonParserTest, ClassType) +{ + model::PYName pyname; + + pyname = queryFile("classes.py", + (odb::query::line_start == 1 && + odb::query::value == "class Base:\n")); + + EXPECT_EQ(pyname.type, "class"); + + pyname = queryFile("classes.py", + (odb::query::line_start == 19 && + odb::query::value == "class Derived(Base):\n")); + + EXPECT_EQ(pyname.type, "class"); + + pyname = queryFile("classes.py", + (odb::query::line_start == 23 && + odb::query::value == "class Derived2(Derived, Base):\n")); + + EXPECT_EQ(pyname.type, "class"); +} + +TEST_F(PythonParserTest, ClassInheritance) +{ + model::PYName pyname; + + model::PYName derived = queryFile("classes.py", + (odb::query::line_start == 19 && + odb::query::type == "class")); + + model::PYName derived2 = queryFile("classes.py", + (odb::query::line_start == 23 && + odb::query::type == "class")); + + pyname = queryFile("classes.py", + (odb::query::line_start == 19 && + odb::query::column_start == 15 && + odb::query::value == "Base")); + + EXPECT_EQ(pyname.type, "baseclass"); + EXPECT_EQ(pyname.parent, derived.id); + + pyname = queryFile("classes.py", + (odb::query::line_start == 23 && + odb::query::column_start == 16 && + odb::query::value == "Derived")); + + EXPECT_EQ(pyname.type, "baseclass"); + EXPECT_EQ(pyname.parent, derived2.id); + + pyname = queryFile("classes.py", + (odb::query::line_start == 23 && + odb::query::column_start == 25 && + odb::query::value == "Base")); + + EXPECT_EQ(pyname.type, "baseclass"); + EXPECT_EQ(pyname.parent, derived2.id); +} + +TEST_F(PythonParserTest, ClassMethod) +{ + model::PYName base = queryFile("classes.py", + (odb::query::line_start == 1 && + odb::query::type == "class")); + + model::PYName foo = queryFile("classes.py", + (odb::query::line_start == 8 && + odb::query::is_definition == true && + odb::query::type == "function")); + + EXPECT_EQ(foo.parent, base.id); + + model::PYName bar = queryFile("classes.py", + (odb::query::line_start == 11 && + odb::query::is_definition == true && + odb::query::type == "function")); + + EXPECT_EQ(bar.parent, base.id); + + model::PYName test = queryFile("classes.py", + (odb::query::line_start == 14 && + odb::query::is_definition == true && + odb::query::type == "function")); + + EXPECT_EQ(test.parent, bar.id); +} + +TEST_F(PythonParserTest, ClassDataMember) +{ + model::PYName pyname; + + model::PYName base = queryFile("classes.py", + (odb::query::line_start == 1 && + odb::query::type == "class")); + + pyname = queryFile("classes.py", + odb::query::value == "DEBUG_MODE = False"); + + EXPECT_EQ(pyname.type, "statement"); + EXPECT_EQ(pyname.parent, base.id); + + pyname = queryFile("classes.py", + odb::query::value == "users = []"); + + EXPECT_EQ(pyname.type, "statement"); + EXPECT_EQ(pyname.parent, base.id); +} + +TEST_F(PythonParserTest, LocalVariable) +{ + model::PYName pyname; + + model::PYName func = queryFile("functions.py", + odb::query::value == "def local_var()"); + + pyname = queryFile("functions.py", + (odb::query::line_start == 97 && + odb::query::value == "a = 2")); + + EXPECT_EQ(pyname.type, "statement"); + EXPECT_EQ(pyname.is_definition, true); + EXPECT_EQ(pyname.parent, func.id); + + pyname = queryFile("functions.py", + (odb::query::line_start == 98 && + odb::query::value == "for i in range(0,10):\n")); + + EXPECT_EQ(pyname.type, "statement"); + EXPECT_EQ(pyname.is_definition, true); + EXPECT_EQ(pyname.parent, func.id); +} + +TEST_F(PythonParserTest, ImportModule) +{ + model::PYName pyname; + + pyname = queryFile("imports.py", + (odb::query::line_start == 1 && + odb::query::value == "import classes")); + + EXPECT_EQ(pyname.is_import, true); + + pyname = queryFile("imports.py", + (odb::query::line_start == 2 && + odb::query::value == "import os")); + + EXPECT_EQ(pyname.is_import, true); + + pyname = queryFile("imports.py", + (odb::query::line_start == 3 && + odb::query::value == "from functions import mul")); + + EXPECT_EQ(pyname.is_import, true); +} + +TEST_F(PythonParserTest, BuiltinVariable) +{ + model::PYName pyname; + + pyname = queryFile("imports.py", + (odb::query::line_start == 2 && + odb::query::value == "import os")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("imports.py", + (odb::query::line_start == 6 && + odb::query::value == "print")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("imports.py", + (odb::query::line_start == 12 && + odb::query::value == "getpid")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("functions.py", + (odb::query::line_start == 85 && + odb::query::value == "str")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("functions.py", + (odb::query::line_start == 85 && + odb::query::value == "List")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("functions.py", + (odb::query::line_start == 98 && + odb::query::value == "range")); + + EXPECT_EQ(pyname.is_builtin, true); +} + +TEST_F(PythonParserTest, ReferenceID) +{ + model::PYName pyname; + + model::PYName func = queryFile("functions.py", odb::query::value == "def mul(a, b)"); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 11); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 14); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 17); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 22); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 26); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 27); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 43); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 44); + EXPECT_EQ(pyname.ref_id, func.id); +} diff --git a/plugins/python/test/src/pythonservicetest.cpp b/plugins/python/test/src/pythonservicetest.cpp new file mode 100644 index 000000000..7f3446870 --- /dev/null +++ b/plugins/python/test/src/pythonservicetest.cpp @@ -0,0 +1,346 @@ +#include +#include +#include +#include + +#include +#include +#include + +using namespace cc; +using namespace cc::service::language; +extern const char* dbConnectionString; + +class PythonServiceTest : public ::testing::Test +{ +public: + PythonServiceTest() : + _db(util::connectDatabase(dbConnectionString)), + _transaction(_db), + _pythonservice(new PythonServiceHandler( + _db, + std::make_shared(), + webserver::ServerContext(std::string(), boost::program_options::variables_map())) + ) + { + loadFile("functions.py"); + loadFile("classes.py"); + loadFile("imports.py"); + } + + AstNodeInfo getAstNodeInfoByPosition(const std::string& filename, int32_t line, int32_t column) + { + AstNodeInfo nodeInfo; + + if (m_files.count(filename) == 0) + { + return nodeInfo; + } + + model::FileId file_id = m_files[filename]; + + service::core::FilePosition filePos; + filePos.file = std::to_string(file_id); + filePos.pos.line = line; + filePos.pos.column = column; + + _pythonservice->getAstNodeInfoByPosition(nodeInfo, filePos); + return nodeInfo; + } + + size_t referenceFinder(const std::vector& references, + const std::vector& positions) + { + size_t c = 0; + for (const AstNodeInfo& info : references) + { + size_t line = info.range.range.startpos.line; + size_t column = info.range.range.startpos.column; + + for (const model::Position& pos : positions) + { + if (pos.line == line && pos.column == column) c++; + } + } + + return c; + } + +private: + std::shared_ptr _db; + util::OdbTransaction _transaction; + std::unordered_map m_files; + void loadFile(const std::string& filename) + { + _transaction([&, this]() { + model::File file = _db->query_value(odb::query::filename == filename); + m_files.emplace(filename, file.id); + }); + } + +protected: + std::shared_ptr _pythonservice; +}; + +TEST_F(PythonServiceTest, AstNodeInfoByPosition) +{ + AstNodeInfo nodeInfo; + + // Simulating click on line 1 column 5 + nodeInfo = getAstNodeInfoByPosition("functions.py", 1, 5); + + EXPECT_EQ(nodeInfo.astNodeValue, "def hello_world()"); + EXPECT_EQ(nodeInfo.symbolType, "function"); + + nodeInfo = getAstNodeInfoByPosition("functions.py", 14, 12); + + EXPECT_EQ(nodeInfo.astNodeValue, "mul"); + EXPECT_EQ(nodeInfo.symbolType, "statement"); + + nodeInfo = getAstNodeInfoByPosition("functions.py", 24, 10); + + EXPECT_EQ(nodeInfo.astNodeValue, "MyLib"); + EXPECT_EQ(nodeInfo.symbolType, "statement"); +} + +TEST_F(PythonServiceTest, NodeProperties) +{ + AstNodeInfo nodeInfo; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 7, 5); + std::map map; + + _pythonservice->getProperties(map, nodeInfo.id); + EXPECT_EQ(nodeInfo.astNodeValue, "def mul(a, b)"); + + EXPECT_EQ(map.count("Full name"), 1); + EXPECT_EQ(map["Full name"], "functions.mul"); + EXPECT_EQ(map.count("Builtin"), 1); + EXPECT_EQ(map["Builtin"], "false"); + EXPECT_EQ(map.count("Function call"), 1); + EXPECT_EQ(map["Function call"], "false"); +} + +TEST_F(PythonServiceTest, NodePropertiesBuiltinCall) +{ + AstNodeInfo nodeInfo; + + nodeInfo = getAstNodeInfoByPosition("imports.py", 12, 17); + EXPECT_EQ(nodeInfo.astNodeValue, "getpid"); + + std::map map; + + _pythonservice->getProperties(map, nodeInfo.id); + EXPECT_EQ(map.count("Full name"), 1); + EXPECT_EQ(map["Full name"], "imports.getpid"); + EXPECT_EQ(map.count("Builtin"), 1); + EXPECT_EQ(map["Builtin"], "true"); + EXPECT_EQ(map.count("Function call"), 1); + EXPECT_EQ(map["Function call"], "true"); +} + +TEST_F(PythonServiceTest, NodeDefinition) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 11, 12); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::DEFINITION, {}); + + EXPECT_EQ(references.size(), 1); + EXPECT_EQ(references[0].symbolType, "function"); + EXPECT_EQ(references[0].astNodeValue, "def mul(a, b)"); + + references = {}; + nodeInfo = getAstNodeInfoByPosition("functions.py", 29, 6); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::DEFINITION, {}); + + EXPECT_EQ(references.size(), 1); + EXPECT_EQ(references[0].symbolType, "statement"); + EXPECT_EQ(references[0].astNodeValue, "mylib = {\n"); + + references = {}; + nodeInfo = getAstNodeInfoByPosition("classes.py", 27, 8); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::DEFINITION, {}); + + EXPECT_EQ(references.size(), 1); + EXPECT_EQ(references[0].symbolType, "class"); + EXPECT_EQ(references[0].astNodeValue, "class Base:\n"); +} + +TEST_F(PythonServiceTest, NodeUsage) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 16, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::USAGE, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "mylib = {\n"); + EXPECT_EQ(references.size(), 2); + + size_t found = referenceFinder(references, {{29, 6}, {46, 11}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeDataMembers) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("classes.py", 1, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::DATA_MEMBER, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "class Base:\n"); + EXPECT_EQ(references.size(), 2); + + size_t found = referenceFinder(references, {{2, 5}, {3, 5}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeLocalVariables) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 96, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::LOCAL_VAR, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def local_var()"); + EXPECT_EQ(references.size(), 3); + + size_t found = referenceFinder(references, {{97, 5}, {98, 5}, {99, 9}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeParent) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 97, 5); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::PARENT, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "a = 2"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{96, 1}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeParentFunction) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("classes.py", 31, 9); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::PARENT_FUNCTION, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "z = 2"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{29, 1}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeCaller) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 13, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::CALLER, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def mul3()"); + EXPECT_EQ(references.size(), 2); + + size_t found = referenceFinder(references, {{28, 6}, {45, 11}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeThisCalls) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 10, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::THIS_CALLS, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def mul2(a, b)"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{11, 12}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeMethod) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("classes.py", 1, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::METHOD, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "class Base:\n"); + EXPECT_EQ(references.size(), 3); + + size_t found = referenceFinder(references, {{5, 5}, {8, 5}, {11, 5}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeBaseClass) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("classes.py", 19, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::BASE_CLASS, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "class Derived(Base):\n"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{1, 1}}); + EXPECT_EQ(found, references.size()); + + // ---------- + + references = {}; + nodeInfo = getAstNodeInfoByPosition("classes.py", 23, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::BASE_CLASS, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "class Derived2(Derived, Base):\n"); + EXPECT_EQ(references.size(), 2); + + found = referenceFinder(references, {{1, 1}, {19, 1}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeFunctionParam) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 4, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::PARAMETER, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def runner(func, param1, param2)"); + EXPECT_EQ(references.size(), 3); + + size_t found = referenceFinder(references, {{4, 12}, {4, 18}, {4, 26}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeAnnotation) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 69, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::ANNOTATION, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def annotation(a, b) -> None"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{69, 24}}); + EXPECT_EQ(found, references.size()); +} diff --git a/plugins/python/test/src/pythontest.cpp b/plugins/python/test/src/pythontest.cpp new file mode 100644 index 000000000..6b1ebbe6a --- /dev/null +++ b/plugins/python/test/src/pythontest.cpp @@ -0,0 +1,20 @@ +#include + +const char* dbConnectionString; + +int main(int argc, char** argv) +{ + if (argc < 3) + { + GTEST_LOG_(FATAL) << "Test arguments missing."; + return 1; + } + + GTEST_LOG_(INFO) << "Testing Python started..."; + system(argv[1]); + + GTEST_LOG_(INFO) << "Using database for tests: " << dbConnectionString; + dbConnectionString = argv[2]; + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/plugins/python/webgui/css/pythonplugin.css b/plugins/python/webgui/css/pythonplugin.css new file mode 100644 index 000000000..4349563fc --- /dev/null +++ b/plugins/python/webgui/css/pythonplugin.css @@ -0,0 +1,23 @@ +.icon-Builtin::before { + content : '\e017' +} + +.icon-Type-hint::before { + content : '\e040' +} + +.icon-Full-name::before { + content: '\e001'; +} + +.icon-Parent::before { + content : '\e034' +} + +.icon-Base-class::before { + content : '\e033' +} + +.icon-Function-call::before { + content : '\e028' +} diff --git a/plugins/python/webgui/js/pythonDiagram.js b/plugins/python/webgui/js/pythonDiagram.js new file mode 100644 index 000000000..e0a16e15f --- /dev/null +++ b/plugins/python/webgui/js/pythonDiagram.js @@ -0,0 +1,126 @@ +require([ + 'dojo/topic', + 'dijit/Menu', + 'dijit/MenuItem', + 'dijit/PopupMenuItem', + 'codecompass/model', + 'codecompass/viewHandler'], +function (topic, Menu, MenuItem, PopupMenuItem, model, viewHandler) { + + model.addService('pythonservice', 'PythonService', LanguageServiceClient); + + var astDiagram = { + id : 'python-ast-diagram', + + getDiagram : function (diagramType, nodeId, callback) { + model.pythonservice.getDiagram(nodeId, diagramType, callback); + }, + + getDiagramLegend : function (diagramType) { + return model.pythonservice.getDiagramLegend(diagramType); + }, + + mouseOverInfo : function (diagramType, nodeId) { + var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); + var range = nodeInfo.range.range; + + return { + fileId : nodeInfo.range.file, + selection : [ + range.startpos.line, + range.startpos.column, + range.endpos.line, + range.endpos.column + ] + }; + } + }; + + viewHandler.registerModule(astDiagram, { + type : viewHandler.moduleType.Diagram + }); + + var fileDiagramHandler = { + id : 'python-file-diagram-handler', + + getDiagram : function (diagramType, nodeId, callback) { + // File path node + if (["d","s","f"].includes(nodeId[0])) { + // When requesting a file diagram, a valid file needs to be used. + // The first character is therefore removed. + nodeId = nodeId.substring(1); + } else { + var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); + nodeId = nodeInfo.range.file; + } + + model.pythonservice.getFileDiagram(nodeId, diagramType, callback); + }, + + getDiagramLegend : function (diagramType) { + return model.pythonservice.getFileDiagramLegend(diagramType); + }, + + mouseOverInfo : function (diagramType, nodeId) { + // File path node + if (["d","s","f"].includes(nodeId[0])) { + return { + fileId : nodeId.substring(1), + selection : [1,1,1,1] + }; + } + + var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); + var range = nodeInfo.range.range; + + return { + fileId : nodeInfo.range.file, + selection : [ + range.startpos.line, + range.startpos.column, + range.endpos.line, + range.endpos.column + ] + }; + } + }; + + viewHandler.registerModule(fileDiagramHandler, { + type : viewHandler.moduleType.Diagram + }); + + var fileDiagrams = { + id : 'python-file-diagrams', + render : function (fileInfo) { + var submenu = new Menu(); + + var diagramTypes = model.pythonservice.getFileDiagramTypes(fileInfo.id); + for (diagramType in diagramTypes) + submenu.addChild(new MenuItem({ + label : diagramType, + type : diagramType, + onClick : function () { + var that = this; + + topic.publish('codecompass/openFile', { fileId : fileInfo.id }); + + topic.publish('codecompass/openDiagram', { + handler : 'python-file-diagram-handler', + diagramType : diagramTypes[that.type], + node : "f" + fileInfo.id + }); + } + })); + + if (Object.keys(diagramTypes).length !== 0) + return new PopupMenuItem({ + label : 'Python Diagrams', + popup : submenu + }); + } + }; + + viewHandler.registerModule(fileDiagrams, { + type : viewHandler.moduleType.FileManagerContextMenu + }); +}); diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js new file mode 100644 index 000000000..f2af28497 --- /dev/null +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -0,0 +1,233 @@ +require([ + 'codecompass/model', + 'codecompass/viewHandler', + 'codecompass/util'], + function (model, viewHandler, util) { + + model.addService('pythonservice', 'PythonService', LanguageServiceClient); + + function createReferenceCountLabel(label, count) { + var parsedLabel = $('
').append($.parseHTML(label)); + parsedLabel.children('span.reference-count').remove(); + parsedLabel.append('(' + count + ')'); + + return parsedLabel.html(); + } + + function createLabel(astNodeInfo) { + var labelValue = astNodeInfo.astNodeValue.trim(); + + if (labelValue.slice(-1) == ':') labelValue = labelValue.substr(0, labelValue.length - 1); + if(astNodeInfo.astNodeType && !labelValue.includes(':')) labelValue += ' : ' + astNodeInfo.astNodeType + ""; + + var label = '' + + astNodeInfo.range.range.startpos.line + ':' + + astNodeInfo.range.range.startpos.column + ': ' + + labelValue + + ''; + + return label; + } + + function loadReferenceNodes(parentNode, nodeInfo, refTypes, scratch) { + var res = []; + var fileGroupsId = []; + + var references = model.pythonservice.getReferences( + nodeInfo.id, + parentNode.refType); + + references.forEach(function (reference) { + if (parentNode.refType === refTypes['Caller'] || + parentNode.refType === refTypes['Usage']) { + + //--- Group nodes by file name ---// + + var fileId = reference.range.file; + if (fileGroupsId[fileId]) + return; + + fileGroupsId[fileId] = parentNode.refType + fileId + reference.id; + + var referenceInFile = references.filter(function (reference) { + return reference.range.file === fileId; + }); + + var fileInfo = model.project.getFileInfo(fileId); + + res.push({ + id : fileGroupsId[fileId], + name : createReferenceCountLabel( + fileInfo.name, referenceInFile.length), + refType : parentNode.refType, + hasChildren : true, + cssClass : util.getIconClass(fileInfo.path), + getChildren : function () { + var that = this; + var res = []; + + referenceInFile.forEach(function (reference) { + if (parentNode.refType === refTypes['Caller'] || + parentNode.refType === refTypes['Usage']) { + res.push({ + id : fileGroupsId[fileId] + reference.id, + name : createLabel(reference), + refType : parentNode.refType, + nodeInfo : reference, + hasChildren : false, + cssClass : null + }); + } + }); + return res; + } + }); + } else { + res.push({ + name : createLabel(reference), + refType : parentNode.refType, + nodeInfo : reference, + hasChildren : false, + cssClass : null + }); + } + }); + + return res; + } + + /** + * This function returns file references children. + * @param parentNode Reference type node in Info Tree. + */ + function loadFileReferenceNodes(parentNode) { + var res = []; + + var references = model.pythonservice.getFileReferences( + parentNode.nodeInfo.id, + parentNode.refType); + + references.forEach(function (reference) { + res.push({ + name : createLabel(reference), + refType : parentNode.refType, + nodeInfo : reference, + hasChildren : false, + cssClass : null + }); + }); + + return res; + } + + function createRootNode(elementInfo) { + var rootLabel + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.symbolType + : 'File') + + ''; + + var rootValue + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.astNodeValue + : elementInfo.name) + + ''; + + var label = + '' + + rootLabel + ': ' + rootValue + + ''; + + return { + id : 'root', + name : label, + cssClass : 'icon-info', + hasChildren : true, + getChildren : function () { + return that._store.query({ parent : 'root' }); + } + }; + } + + var pythonInfoTree = { + id: 'python-info-tree', + render : function (elementInfo) { + var ret = []; + + ret.push(createRootNode(elementInfo)); + + if (elementInfo instanceof AstNodeInfo) { + //--- Properties ---// + + var props = model.pythonservice.getProperties(elementInfo.id); + + for (var propName in props) { + var label + = '' + propName + ': ' + + '' + props[propName] + ''; + + ret.push({ + name : label, + parent : 'root', + nodeInfo : elementInfo, + cssClass : 'icon-' + propName.replace(/ /g, '-'), + hasChildren : false + }); + } + + //--- References ---// + + var refTypes = model.pythonservice.getReferenceTypes(elementInfo.id); + for (var refType in refTypes) { + var refCount = + model.pythonservice.getReferenceCount(elementInfo.id, refTypes[refType]); + + if (refCount) + ret.push({ + name : createReferenceCountLabel(refType, refCount), + parent : 'root', + refType : refTypes[refType], + cssClass : 'icon-' + refType.replace(/ /g, '-'), + hasChildren : true, + getChildren : function () { + return loadReferenceNodes(this, elementInfo, refTypes); + } + }); + }; + + } else if (elementInfo instanceof FileInfo) { + + //--- File references ---// + + var refTypes = model.pythonservice.getFileReferenceTypes(elementInfo.id); + for (var refType in refTypes) { + var refCount = model.pythonservice.getFileReferenceCount( + elementInfo.id, refTypes[refType]); + + if (refCount) + ret.push({ + name : createReferenceCountLabel(refType, refCount), + parent : 'root', + nodeInfo : elementInfo, + refType : refTypes[refType], + cssClass : 'icon-' + refType.replace(/ /g, '-'), + hasChildren : true, + getChildren : function () { + return loadFileReferenceNodes(this); + } + }); + }; + + } + + return ret; + } + }; + + viewHandler.registerModule(pythonInfoTree, { + type : viewHandler.moduleType.InfoTree, + service : model.pythonservice + }); + }); diff --git a/plugins/python/webgui/js/pythonMenu.js b/plugins/python/webgui/js/pythonMenu.js new file mode 100644 index 000000000..d3487fe3f --- /dev/null +++ b/plugins/python/webgui/js/pythonMenu.js @@ -0,0 +1,151 @@ +require([ + 'dojo/topic', + 'dijit/Menu', + 'dijit/MenuItem', + 'dijit/PopupMenuItem', + 'codecompass/astHelper', + 'codecompass/model', + 'codecompass/urlHandler', + 'codecompass/viewHandler'], +function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, urlHandler, viewHandler) { + + model.addService('pythonservice', 'PythonService', LanguageServiceClient); + + var getdefintion = { + id : 'python-text-getdefintion', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Jump to definition', + accelKey : 'ctrl - click', + onClick : function () { + if (!nodeInfo || !fileInfo) + return; + + var languageService = model.getLanguageService(fileInfo.type); + astHelper.jumpToDef(nodeInfo.id, model.pythonservice); + + if (window.gtag) { + window.gtag ('event', 'jump_to_def', { + 'event_category' : urlHandler.getState('wsid'), + 'event_label' : urlHandler.getFileInfo().name + + ': ' + + nodeInfo.astNodeValue + }); + } + } + }); + } + }; + + viewHandler.registerModule(getdefintion, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var infoTree = { + id : 'python-text-infotree', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Info Tree', + onClick : function () { + if (!nodeInfo || !fileInfo) + return; + + topic.publish('codecompass/infotree', { + fileType : fileInfo.type, + elementInfo : nodeInfo + }); + + if (window.gtag) { + window.gtag ('event', 'info_tree', { + 'event_category' : urlHandler.getState('wsid'), + 'event_label' : urlHandler.getFileInfo().name + + ': ' + + nodeInfo.astNodeValue + }); + } + } + }); + } + }; + + viewHandler.registerModule(infoTree, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var infobox = { + id : 'python-text-infobox', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Documentation', + onClick : function () { + topic.publish('codecompass/documentation', { + fileType : fileInfo.type, + elementInfo : nodeInfo + }); + + if (window.gtag) { + window.gtag ('event', 'documentation', { + 'event_category' : urlHandler.getState('wsid'), + 'event_label' : urlHandler.getFileInfo().name + + ': ' + + nodeInfo.astNodeValue + }); + } + } + }); + } + }; + + viewHandler.registerModule(infobox, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var diagrams = { + id : 'python-text-diagrams', + render : function (nodeInfo, fileInfo) { + if (!nodeInfo || !fileInfo) + return; + + var submenu = new Menu(); + + var diagramTypes = model.pythonservice.getDiagramTypes(nodeInfo.id); + for (diagramType in diagramTypes) + submenu.addChild(new MenuItem({ + label : diagramType, + type : diagramType, + onClick : function () { + var that = this; + + topic.publish('codecompass/openDiagram', { + handler : 'python-ast-diagram', + diagramType : diagramTypes[that.type], + node : nodeInfo.id + }); + } + })); + + submenu.addChild(new MenuItem({ + label : "CodeBites", + onClick : function () { + topic.publish('codecompass/codebites', { + node : nodeInfo + }); + } + })); + + if (Object.keys(diagramTypes).length !== 0) + return new PopupMenuItem({ + label : 'Diagrams', + popup : submenu + }); + } + }; + + viewHandler.registerModule(diagrams, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); +}); diff --git a/service/workspace/include/workspaceservice/workspaceservice.h b/service/workspace/include/workspaceservice/workspaceservice.h index 8926eac2f..9361a67b2 100644 --- a/service/workspace/include/workspaceservice/workspaceservice.h +++ b/service/workspace/include/workspaceservice/workspaceservice.h @@ -2,6 +2,7 @@ #define CC_SERVICE_WORKSPACE_WORKSPACESERVICE_H #include +#include namespace cc { diff --git a/util/include/util/graph.h b/util/include/util/graph.h index c58ae5914..0285014c2 100644 --- a/util/include/util/graph.h +++ b/util/include/util/graph.h @@ -27,7 +27,7 @@ struct GraphPimpl; class Graph { public: - enum Format {DOT, SVG}; + enum Format {DOT, SVG, CAIRO_SVG}; typedef std::string Node; typedef std::string Edge; diff --git a/util/include/util/util.h b/util/include/util/util.h index 1af6f90d9..7f8d78b37 100644 --- a/util/include/util/util.h +++ b/util/include/util/util.h @@ -29,6 +29,10 @@ std::string textRange( */ std::string escapeHtml(const std::string& str_); +inline const char* boolToString(bool b) { + return b ? "true" : "false"; +} + } } diff --git a/util/src/dynamiclibrary.cpp b/util/src/dynamiclibrary.cpp index 251af2268..4d42c64ef 100644 --- a/util/src/dynamiclibrary.cpp +++ b/util/src/dynamiclibrary.cpp @@ -37,7 +37,7 @@ DynamicLibrary::DynamicLibrary(const std::string& path_) throw std::runtime_error(ss.str()); } #else - _handle = ::dlopen(path_.c_str(), RTLD_NOW); + _handle = ::dlopen(path_.c_str(), RTLD_NOW | RTLD_GLOBAL); if (!_handle) { const char *dlError = ::dlerror(); diff --git a/util/src/graph.cpp b/util/src/graph.cpp index 7d15c5936..155106201 100644 --- a/util/src/graph.cpp +++ b/util/src/graph.cpp @@ -278,10 +278,26 @@ std::string Graph::output(Graph::Format format_) const gvLayout(_graphPimpl->_gvc, _graphPimpl->_graph, "dot"); + const char* render_format; + switch (format_) { + case Graph::DOT: + render_format = "dot"; + break; + case Graph::SVG: + render_format = "svg"; + break; + case Graph::CAIRO_SVG: + render_format = "svg:cairo"; + break; + default: + __builtin_unreachable(); + break; + } + gvRenderData( _graphPimpl->_gvc, _graphPimpl->_graph, - format_ == Graph::DOT ? "dot" : "svg", + render_format, result, length); diff --git a/webgui/scripts/codecompass/view/component/Text.js b/webgui/scripts/codecompass/view/component/Text.js index 473c820e1..8eacba345 100644 --- a/webgui/scripts/codecompass/view/component/Text.js +++ b/webgui/scripts/codecompass/view/component/Text.js @@ -341,6 +341,11 @@ function (declare, domClass, dom, style, query, topic, ContentPane, Dialog, this.set('content', fileContent); this.set('header', this._fileInfo); + if(this._fileInfo.type == "PY") + { + this._codeMirror.setOption("mode", 'text/x-python'); + } + this._getSyntaxHighlight(this._fileInfo); if (window.gtag) {