From 6e0abaa2f71e9a21cd003d9beed6477f57dfd9fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 12 Oct 2023 17:40:47 +0200 Subject: [PATCH 001/156] Added python plugin based on dummy and cpp --- plugins/python/CMakeLists.txt | 3 + plugins/python/parser/CMakeLists.txt | 12 + .../include/pythonparser/pythonparser.h | 25 ++ plugins/python/parser/src/pythonparser.cpp | 63 +++++ plugins/python/service/CMakeLists.txt | 26 ++ .../service/include/service/pythonservice.h | 261 ++++++++++++++++++ plugins/python/service/src/plugin.cpp | 33 +++ plugins/python/service/src/pythonservice.cpp | 21 ++ 8 files changed, 444 insertions(+) create mode 100644 plugins/python/CMakeLists.txt create mode 100644 plugins/python/parser/CMakeLists.txt create mode 100644 plugins/python/parser/include/pythonparser/pythonparser.h create mode 100644 plugins/python/parser/src/pythonparser.cpp create mode 100644 plugins/python/service/CMakeLists.txt create mode 100644 plugins/python/service/include/service/pythonservice.h create mode 100644 plugins/python/service/src/plugin.cpp create mode 100644 plugins/python/service/src/pythonservice.cpp diff --git a/plugins/python/CMakeLists.txt b/plugins/python/CMakeLists.txt new file mode 100644 index 000000000..da5d57725 --- /dev/null +++ b/plugins/python/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(parser) +#add_subdirectory(test) +add_subdirectory(service) diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt new file mode 100644 index 000000000..55c20f20a --- /dev/null +++ b/plugins/python/parser/CMakeLists.txt @@ -0,0 +1,12 @@ +include_directories( + include + ${PROJECT_SOURCE_DIR}/model/include + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_SOURCE_DIR}/parser/include) + +add_library(pythonparser SHARED + src/pythonparser.cpp) + +target_compile_options(pythonparser PUBLIC -Wno-unknown-pragmas) + +install(TARGETS pythonparser DESTINATION ${INSTALL_PARSER_DIR}) diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h new file mode 100644 index 000000000..8ae0595a1 --- /dev/null +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -0,0 +1,25 @@ +#ifndef CC_PARSER_PYTHONPARSER_H +#define CC_PARSER_PYTHONPARSER_H + +#include +#include + +namespace cc +{ +namespace parser +{ + +class PythonParser : public AbstractParser +{ +public: + PythonParser(ParserContext& ctx_); + virtual ~PythonParser(); + virtual bool parse() override; +private: + bool accept(const std::string& path_); +}; + +} // parser +} // cc + +#endif // CC_PARSER_PYTHONPARSER_H diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp new file mode 100644 index 000000000..dc37d7400 --- /dev/null +++ b/plugins/python/parser/src/pythonparser.cpp @@ -0,0 +1,63 @@ +#include + +#include + +#include + +#include + +namespace cc +{ +namespace parser +{ + +PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) +{ +} + +bool PythonParser::accept(const std::string& path_) +{ + std::string ext = boost::filesystem::extension(path_); + return ext == ".dummy"; +} + +bool PythonParser::parse() +{ + for(std::string path : _ctx.options["input"].as>()) + { + if(accept(path)) + { + LOG(info) << "DummyParser parse path: " << path; + } + } + return true; +} + +PythonParser::~PythonParser() +{ +} + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ + boost::program_options::options_description getOptions() + { + boost::program_options::options_description description("Python Plugin"); + + description.add_options() + ("dummy-arg", po::value()->default_value("Dummy arg"), + "This argument will be used by the dummy parser."); + + return description; + } + + std::shared_ptr make(ParserContext& ctx_) + { + return std::make_shared(ctx_); + } +} +#pragma clang diagnostic pop + +} // parser +} // cc diff --git a/plugins/python/service/CMakeLists.txt b/plugins/python/service/CMakeLists.txt new file mode 100644 index 000000000..f755ba09c --- /dev/null +++ b/plugins/python/service/CMakeLists.txt @@ -0,0 +1,26 @@ +include_directories( + include + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_SOURCE_DIR}/webserver/include + ${PROJECT_BINARY_DIR}/service/language/gen-cpp + ${PROJECT_BINARY_DIR}/service/project/gen-cpp + ${PROJECT_SOURCE_DIR}/service/project/include) + +include_directories(SYSTEM + ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}) + +add_library(pythonservice SHARED + src/pythonservice.cpp + src/plugin.cpp) + +target_compile_options(pythonservice PUBLIC -Wno-unknown-pragmas) + +target_link_libraries(pythonservice + util + projectservice + languagethrift + ${THRIFT_LIBTHRIFT_LIBRARIES} + ${ODB_LIBRARIES}) + +install(TARGETS pythonservice DESTINATION ${INSTALL_SERVICE_DIR}) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h new file mode 100644 index 000000000..1040cfed9 --- /dev/null +++ b/plugins/python/service/include/service/pythonservice.h @@ -0,0 +1,261 @@ +#ifndef CC_SERVICE_PYTHON_PYTHONSERVICE_H +#define CC_SERVICE_PYTHON_PYTHONSERVICE_H + +#include +#include + +#include + +#include +#include +#include + +#include + +namespace cc +{ +namespace service +{ +namespace language +{ + +class PythonServiceHandler : virtual public LanguageServiceIf +{ +public: + PythonServiceHandler( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_); + + void getFileTypes(std::vector& return_) override; + + void getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) override; + + void getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) override; + + void getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) override; + + void getDocumentation( + std::string& return_, + const core::AstNodeId& astNodeId_) override; + + void getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) override; + + void getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getFileDiagramTypes( + std::map& return_, + const core::FileId& fileId_) override; + + void getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) override; + + void getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) override; + + void getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector& tags_) override; + + std::int32_t getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) override; + + void getReferencesInFile( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const core::FileId& fileId_, + const std::vector& tags_) override; + + void getReferencesPage( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::int32_t pageSize_, + const std::int32_t pageNo_) override; + + void getFileReferenceTypes( + std::map& return_, + const core::FileId& fileId_) override; + + void getFileReferences( + std::vector& return_, + const core::FileId& fileId_, + const std::int32_t referenceId_) override; + + std::int32_t getFileReferenceCount( + const core::FileId& fileId_, + const std::int32_t referenceId_) override; + + void getSyntaxHighlight( + std::vector& return_, + const core::FileRange& range_) override; + +private: + enum ReferenceType + { + DEFINITION, /*!< By this option the definition(s) of the AST node can be + queried. However according to the "one definition rule" a named entity + can have only one definition, in a parsing several definitions might be + available. This is the case when the project is built for several targets + and in the different builds different definitions are defined for an + entity (e.g. because of an #ifdef section). */ + + DECLARATION, /*!< By this options the declaration(s) of the AST node can be + queried. */ + + USAGE, /*!< By this option the usages of the AST node can be queried, i.e. + the nodes of which the entity hash is identical to the queried one. */ + + THIS_CALLS, /*!< Get function calls in a function. WARNING: If the + definition of the AST node is not unique then it returns the callees of + one of them. */ + + CALLS_OF_THIS, /*!< Get calls of a function. */ + + CALLEE, /*!< Get called functions definitions. WARNING: If the definition of + the AST node is not unique then it returns the callees of one of them. */ + + CALLER, /*!< Get caller functions. */ + + VIRTUAL_CALL, /*!< A function may be used virtually on a base type object. + The exact type of the object is based on dynamic information, which can't + be determined statically. Weak usage returns these possible calls. */ + + FUNC_PTR_CALL, /*!< Functions can be assigned to function pointers which + can be invoked later. This option returns these invocations. */ + + PARAMETER, /*!< This option returns the parameters of a function. */ + + LOCAL_VAR, /*!< This option returns the local variables of a function. */ + + RETURN_TYPE, /*!< This option returns the return type of a function. */ + + OVERRIDE, /*!< This option returns the functions which the given function + overrides. */ + + OVERRIDDEN_BY, /*!< This option returns the overrides of a function. */ + + READ, /*!< This option returns the places where a variable is read. */ + + WRITE, /*!< This option returns the places where a variable is written. */ + + TYPE, /*!< This option returns the type of a variable. */ + + ALIAS, /*!< Types may have aliases, e.g. by typedefs. */ + + INHERIT_FROM, /*!< Types from which the queried type inherits. */ + + INHERIT_BY, /*!< Types by which the queried type is inherited. */ + + DATA_MEMBER, /*!< Data members of a class. */ + + METHOD, /*!< Members of a class. */ + + FRIEND, /*!< The friends of a class. */ + + UNDERLYING_TYPE, /*!< Underlying type of a typedef. */ + + ENUM_CONSTANTS, /*!< Enum constants. */ + + EXPANSION, /*!< Macro expansion. */ + + UNDEFINITION, /*!< Macro undefinition. */ + }; + + enum FileReferenceType + { + INCLUDES, /*!< Included source files in the current source file after the + inclusion directive. */ + + TYPES, /*!< User defined data types such as classes, structs etc. */ + + FUNCTIONS, /*!< Functions in the current source file. */ + + MACROS, /*!< Macros in the current source file. */ + }; + + enum DiagramType + { + FUNCTION_CALL, /*!< In the function call diagram the nodes are functions and + the edges are the function calls between them. The diagram also displays + some dynamic information such as virtual function calls. */ + + DETAILED_CLASS, /*!< This is a classical UML class diagram for the selected + class and its direct children and parents. The nodes contain the methods + and member variables with their visibility. */ + + CLASS_OVERVIEW, /*!< This is a class diagram which contains all classes + which inherit from the current one, and all parents from which the + current one inherits. The methods and member variables are node included + in the nodes, but the type of the member variables are indicated as + aggregation relationship. */ + + CLASS_COLLABORATION, /*!< This returns a class collaboration diagram + which shows the individual class members and their inheritance + hierarchy. */ + + COMPONENT_USERS, /*!< Component users diagram for source file S shows which + source files depend on S through the interfaces S provides. */ + + EXTERNAL_DEPENDENCY, /*!< This diagram shows the module which directory + depends on. The "depends on" diagram on module A traverses the + subdirectories of module A and shows all directories that contain files + that any of the source files in A includes. */ + + EXTERNAL_USERS, /*!< This diagram shows directories (modules) that are + users of the queried module. */ + + INCLUDE_DEPENDENCY, /*!< This diagram shows of the `#include` file + dependencies. */ + + INTERFACE, /*!< Interface diagram shows the used and provided interfaces of + a source code file and shows linking information. */ + + SUBSYSTEM_DEPENDENCY, /*!< This diagram shows the directories relationship + between the subdirectories of the queried module. This diagram is useful + to understand the relationships of the subdirectories (submodules) + of a module. */ + }; + + std::shared_ptr _db; + util::OdbTransaction _transaction; + + std::shared_ptr _datadir; + const cc::webserver::ServerContext& _context; +}; + +} // language +} // service +} // cc + +#endif // CC_SERVICE_PYTHON_PYTHONSERVICE_H diff --git a/plugins/python/service/src/plugin.cpp b/plugins/python/service/src/plugin.cpp new file mode 100644 index 000000000..0826da989 --- /dev/null +++ b/plugins/python/service/src/plugin.cpp @@ -0,0 +1,33 @@ +#include + +#include + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ + boost::program_options::options_description getOptions() + { + namespace po = boost::program_options; + + po::options_description description("Python Plugin"); + + description.add_options() + ("dummy-result", po::value()->default_value("Dummy result"), + "This value will be returned by the dummy service."); + + return description; + } + + void registerPlugin( + const cc::webserver::ServerContext& context_, + cc::webserver::PluginHandler* pluginHandler_) + { + cc::webserver::registerPluginSimple( + context_, + pluginHandler_, + CODECOMPASS_LANGUAGE_SERVICE_FACTORY_WITH_CFG(Python), + "PythonService"); + } +} +#pragma clang diagnostic pop diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp new file mode 100644 index 000000000..c0c6a094e --- /dev/null +++ b/plugins/python/service/src/pythonservice.cpp @@ -0,0 +1,21 @@ +#include +#include + +namespace cc +{ +namespace service +{ +namespace language +{ + +PythonServiceHandler::PythonServiceHandler( + std::shared_ptr db_, + std::shared_ptr /*datadir_*/, + const cc::webserver::ServerContext& context_) + : _db(db_), _transaction(db_), _context(context_) +{ +} + +} // python +} // service +} // cc From 11c9ac75c6aeeed695df13db5d95c9b66cb5283e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 12 Oct 2023 23:53:43 +0200 Subject: [PATCH 002/156] PythonService basic implementations --- plugins/python/service/src/pythonservice.cpp | 159 ++++++++++++++++++- 1 file changed, 157 insertions(+), 2 deletions(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index c0c6a094e..27c75a08f 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -12,10 +12,165 @@ PythonServiceHandler::PythonServiceHandler( std::shared_ptr db_, std::shared_ptr /*datadir_*/, const cc::webserver::ServerContext& context_) - : _db(db_), _transaction(db_), _context(context_) + : _db(db_), _transaction(db_), _context(context_) {} + +void PythonServiceHandler::getFileTypes( + std::vector& return_) +{ + return; +} + +void PythonServiceHandler::getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) +{ + return; +} + +void PythonServiceHandler::getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) +{ + return; +} + +void PythonServiceHandler::getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) +{ + return; +} + +void PythonServiceHandler::getDocumentation( + std::string& return_, + const core::AstNodeId& astNodeId_) +{ + return; +} + +void PythonServiceHandler::getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) +{ + return; +} + +void PythonServiceHandler::getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) +{ + return; +} + +void PythonServiceHandler::getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) +{ + return; +} + +void PythonServiceHandler::getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) +{ + return; +} + +void PythonServiceHandler::getFileDiagramTypes( + std::map& return_, + const core::FileId& fileId_) +{ + return; +} + +void PythonServiceHandler::getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) +{ + return; +} + +void PythonServiceHandler::getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) +{ + return; +} + +void PythonServiceHandler::getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) +{ + return; +} + +void PythonServiceHandler::getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector& tags_) +{ + return; +} + +std::int32_t PythonServiceHandler::getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) +{ + return 0; +} + +void PythonServiceHandler::getReferencesInFile( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const core::FileId& fileId_, + const std::vector& tags_) +{ + return; +} + +void PythonServiceHandler::getReferencesPage( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::int32_t pageSize_, + const std::int32_t pageNo_) +{ + return; +} + +void PythonServiceHandler::getFileReferenceTypes( + std::map& return_, + const core::FileId& fileId_) +{ + return; +} + +void PythonServiceHandler::getFileReferences( + std::vector& return_, + const core::FileId& fileId_, + const std::int32_t referenceId_) +{ + return; +} + +std::int32_t PythonServiceHandler::getFileReferenceCount( + const core::FileId& fileId_, + const std::int32_t referenceId_) +{ + return 0; +} + +void PythonServiceHandler::getSyntaxHighlight( + std::vector& return_, + const core::FileRange& range_) { + return; } -} // python +} // language } // service } // cc From 95964528715650bb54063b853b5f572a8dcf090f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 13 Oct 2023 20:40:32 +0200 Subject: [PATCH 003/156] Added PythonService file types --- plugins/python/service/src/pythonservice.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 27c75a08f..605e9ed1a 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -17,6 +17,8 @@ PythonServiceHandler::PythonServiceHandler( void PythonServiceHandler::getFileTypes( std::vector& return_) { + return_.push_back("PY"); + return_.push_back("Dir"); return; } From 75224afd2dade05475e11adfc446164a36d35ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 16 Oct 2023 19:57:28 +0200 Subject: [PATCH 004/156] Python plugin webgui --- plugins/python/CMakeLists.txt | 1 + plugins/python/webgui/js/pythonInfoTree.js | 8 ++++++++ plugins/python/webgui/js/pythonMenu.js | 12 ++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 plugins/python/webgui/js/pythonInfoTree.js create mode 100644 plugins/python/webgui/js/pythonMenu.js diff --git a/plugins/python/CMakeLists.txt b/plugins/python/CMakeLists.txt index da5d57725..d7aed06be 100644 --- a/plugins/python/CMakeLists.txt +++ b/plugins/python/CMakeLists.txt @@ -1,3 +1,4 @@ add_subdirectory(parser) #add_subdirectory(test) add_subdirectory(service) +install_webplugin(webgui) diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js new file mode 100644 index 000000000..5d9452dda --- /dev/null +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -0,0 +1,8 @@ +require([ + 'codecompass/model', + 'codecompass/viewHandler', + 'codecompass/util'], +function (model, viewHandler, util) { + model.addService('cppservice', 'CppService', LanguageServiceClient); +}); + \ No newline at end of file diff --git a/plugins/python/webgui/js/pythonMenu.js b/plugins/python/webgui/js/pythonMenu.js new file mode 100644 index 000000000..28bfc9a2b --- /dev/null +++ b/plugins/python/webgui/js/pythonMenu.js @@ -0,0 +1,12 @@ +require([ + 'dojo/topic', + 'dijit/Menu', + 'dijit/MenuItem', + 'dijit/PopupMenuItem', + 'codecompass/astHelper', + 'codecompass/model', + 'codecompass/urlHandler', + 'codecompass/viewHandler'], +function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, urlHandler, viewHandler) { + model.addService('pythonservice', 'PythonService', LanguageServiceClient); +}); \ No newline at end of file From 4bf7c7e335ca540a35806296725883b91bceb85a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 16 Oct 2023 20:05:31 +0200 Subject: [PATCH 005/156] Python plugin webgui infotree --- plugins/python/webgui/js/pythonInfoTree.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index 5d9452dda..861896476 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -3,6 +3,6 @@ require([ 'codecompass/viewHandler', 'codecompass/util'], function (model, viewHandler, util) { - model.addService('cppservice', 'CppService', LanguageServiceClient); + model.addService('pythonservice', 'PythonService', LanguageServiceClient); }); \ No newline at end of file From e5a29732a497651d5d6587eecd27af0db6151234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 19 Oct 2023 17:46:38 +0200 Subject: [PATCH 006/156] Added Python InfoTree and Menu --- plugins/python/webgui/js/pythonInfoTree.js | 388 ++++++++++++++++++++- plugins/python/webgui/js/pythonMenu.js | 96 ++++- 2 files changed, 481 insertions(+), 3 deletions(-) diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index 861896476..122cf99c0 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -2,7 +2,391 @@ require([ 'codecompass/model', 'codecompass/viewHandler', 'codecompass/util'], -function (model, viewHandler, util) { + function (model, viewHandler, util) { + model.addService('pythonservice', 'PythonService', LanguageServiceClient); -}); + + function createTagLabels(tags) { + var label = ''; + + if (!tags) + return label; + + if (tags.indexOf('static') > -1) + label += 'S'; + if (tags.indexOf('constructor') > -1) + label += 'C'; + if (tags.indexOf('destructor') > -1) + label += 'D'; + if (tags.indexOf('implicit') > -1) + label += 'I'; + if (tags.indexOf('inherited') > -1) + label += 'I'; + if (tags.indexOf('virtual') > -1) + label += 'V'; + if (tags.indexOf('global') > -1) + label += 'G'; + + return label; + } + + function createReferenceCountLabel(label, count) { + var parsedLabel = $('
').append($.parseHTML(label)); + parsedLabel.children('span.reference-count').remove(); + parsedLabel.append('(' + count + ')'); + + return parsedLabel.html(); + } + + function createLabel(astNodeInfo) { + var labelClass = ''; + + if (astNodeInfo.tags.indexOf('implicit') > -1) + labelClass = 'label-implicit'; + + var labelValue = astNodeInfo.astNodeValue; + + // Create dom node for return type of a function and place it at the end of + // signature. + if (astNodeInfo.symbolType === 'Function') { + var init = labelValue.slice(0, labelValue.indexOf('(')); + var returnTypeEnd = init.lastIndexOf(' '); + + //--- Constructor, destructor doesn't have return type ---// + + if (returnTypeEnd !== -1) { + var funcSignature = init.slice(returnTypeEnd); + + labelValue = funcSignature + + ' : ' + + init.slice(0, returnTypeEnd) + + ""; + } + } + + var label = createTagLabels(astNodeInfo.tags) + + '' + + astNodeInfo.range.range.startpos.line + ':' + + astNodeInfo.range.range.startpos.column + ': ' + + labelValue + + ''; + + return label; + } + + function getCssClass(astNodeInfo) { + var tags = astNodeInfo.tags; + + return tags.indexOf('public') > -1 ? 'icon-visibility icon-public' : + tags.indexOf('private') > -1 ? 'icon-visibility icon-private' : + tags.indexOf('protected') > -1 ? 'icon-visibility icon-protected' : + null; + } + + function groupReferencesByVisibilities(references, parentNode, nodeInfo) { + var res = []; + var visibilities = ['public', 'private', 'protected']; + + visibilities.forEach(function (visibility) { + var nodes = references.filter(function (reference) { + return reference.tags.indexOf(visibility) > -1; + }); + + if (!nodes.length) + return; + + res.push({ + id : nodeInfo.id + visibility + parentNode.refType, + name : createReferenceCountLabel(visibility, nodes.length), + refType : parentNode.refType, + hasChildren : true, + cssClass : 'icon-visibility icon-' + visibility, + getChildren : function () { + var res = []; + + nodes.forEach(function (reference) { + res.push({ + id : visibility + reference.id, + name : createLabel(reference), + refType : parentNode.refType, + nodeInfo : reference, + hasChildren : false, + cssClass : getCssClass(reference) + }); + }); + + return res; + } + }); + }); + + return res; + } + + function loadReferenceNodes(parentNode, nodeInfo, refTypes, scratch) { + var res = []; + var fileGroupsId = []; + + scratch = scratch || {}; + + var references = model.pythonservice.getReferences( + nodeInfo.id, + parentNode.refType); + + if (parentNode.refType === refTypes['Method'] || + parentNode.refType === refTypes['Data member']) + return groupReferencesByVisibilities(references, parentNode, nodeInfo); + + references.forEach(function (reference) { + if (parentNode.refType === refTypes['Caller'] || + parentNode.refType === refTypes['Usage']) { + + //--- Group nodes by file name ---// + + var fileId = reference.range.file; + if (fileGroupsId[fileId]) + return; + + fileGroupsId[fileId] = parentNode.refType + fileId + reference.id; + + var referenceInFile = references.filter(function (reference) { + return reference.range.file === fileId; + }); + + var fileInfo = model.project.getFileInfo(fileId); + + if (parentNode.refType === refTypes['Caller']) { + scratch.visitedNodeIDs = + (scratch.visitedNodeIDs || []).concat(nodeInfo.id); + } + + res.push({ + id : fileGroupsId[fileId], + name : createReferenceCountLabel( + fileInfo.name, referenceInFile.length), + refType : parentNode.refType, + hasChildren : true, + cssClass : util.getIconClass(fileInfo.path), + getChildren : function () { + var that = this; + var res = []; + + referenceInFile.forEach(function (reference) { + if (parentNode.refType === refTypes['Caller']) { + var showChildren = + scratch.visitedNodeIDs.indexOf(reference.id) == -1; + res.push({ + id : reference.id, + name : createLabel(reference), + nodeInfo : reference, + refType : parentNode.refType, + cssClass : 'icon icon-Method', + hasChildren : showChildren, + getChildren : showChildren + ? function () { + var res = []; + + //--- Recursive Node ---// + + var refCount = model.pythonservice.getReferenceCount( + reference.id, parentNode.refType); + + if (refCount) + res.push({ + id : 'Caller-' + reference.id, + name : createReferenceCountLabel( + parentNode.name, refCount), + nodeInfo : reference, + refType : parentNode.refType, + cssClass : parentNode.cssClass, + hasChildren : true, + getChildren : function () { + return loadReferenceNodes(this, reference, refTypes, scratch); + } + }); + + //--- Call ---// + + var calls = model.pythonservice.getReferences( + this.nodeInfo.id, + refTypes['This calls']); + + calls.forEach(function (call) { + if (call.entityHash === nodeInfo.entityHash) + res.push({ + name : createLabel(call), + refType : parentNode.refType, + nodeInfo : call, + hasChildren : false, + cssClass : getCssClass(call) + }); + }); + return res; + } + : undefined + }); + } else if (parentNode.refType === refTypes['Usage']) { + res.push({ + id : fileGroupsId[fileId] + reference.id, + name : createLabel(reference), + refType : parentNode.refType, + nodeInfo : reference, + hasChildren : false, + cssClass : getCssClass(reference) + }); + } + }); + return res; + } + }); + } else { + res.push({ + name : createLabel(reference), + refType : parentNode.refType, + nodeInfo : reference, + hasChildren : false, + cssClass : getCssClass(reference) + }); + } + }); + + return res; + } + + /** + * This function returns file references children. + * @param parentNode Reference type node in Info Tree. + */ + function loadFileReferenceNodes(parentNode) { + var res = []; + + var references = model.pythonservice.getFileReferences( + parentNode.nodeInfo.id, + parentNode.refType); + + references.forEach(function (reference) { + res.push({ + name : createLabel(reference), + refType : parentNode.refType, + nodeInfo : reference, + hasChildren : false, + cssClass : getCssClass(reference) + }); + }); + + return res; + } + + function createRootNode(elementInfo) { + var rootLabel + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.symbolType + : 'File') + + ''; + + var rootValue + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.astNodeValue + : elementInfo.name) + + ''; + + var label = createTagLabels(elementInfo.tags) + + '' + + rootLabel + ': ' + rootValue + + ''; + + return { + id : 'root', + name : label, + cssClass : 'icon-info', + hasChildren : true, + getChildren : function () { + return that._store.query({ parent : 'root' }); + } + }; + } + + var pythonInfoTree = { + id: 'python-info-tree', + render : function (elementInfo) { + var ret = []; + + ret.push(createRootNode(elementInfo)); + + if (elementInfo instanceof AstNodeInfo) { + //--- Properties ---// + + var props = model.pythonservice.getProperties(elementInfo.id); + + for (var propName in props) { + var propId = propName.replace(/ /g, '-'); + var label + = '' + propName + ': ' + + '' + props[propName] + ''; + + ret.push({ + name : label, + parent : 'root', + nodeInfo : elementInfo, + cssClass : 'icon-' + propId, + hasChildren : false + }); + } + + //--- References ---// + + var refTypes = model.pythonservice.getReferenceTypes(elementInfo.id); + for (var refType in refTypes) { + var refCount = + model.pythonservice.getReferenceCount(elementInfo.id, refTypes[refType]); + + if (refCount) + ret.push({ + name : createReferenceCountLabel(refType, refCount), + parent : 'root', + refType : refTypes[refType], + cssClass : 'icon-' + refType.replace(/ /g, '-'), + hasChildren : true, + getChildren : function () { + return loadReferenceNodes(this, elementInfo, refTypes); + } + }); + }; + + } else if (elementInfo instanceof FileInfo) { + + //--- File references ---// + + var refTypes = model.pythonservice.getFileReferenceTypes(elementInfo.id); + for (var refType in refTypes) { + var refCount = model.pythonservice.getFileReferenceCount( + elementInfo.id, refTypes[refType]); + + if (refCount) + ret.push({ + name : createReferenceCountLabel(refType, refCount), + parent : 'root', + nodeInfo : elementInfo, + refType : refTypes[refType], + cssClass : 'icon-' + refType.replace(/ /g, '-'), + hasChildren : true, + getChildren : function () { + return loadFileReferenceNodes(this); + } + }); + }; + + } + + return ret; + } + }; + + viewHandler.registerModule(pythonInfoTree, { + type : viewHandler.moduleType.InfoTree, + service : model.pythonservice + }); + }); \ No newline at end of file diff --git a/plugins/python/webgui/js/pythonMenu.js b/plugins/python/webgui/js/pythonMenu.js index 28bfc9a2b..d87d6535e 100644 --- a/plugins/python/webgui/js/pythonMenu.js +++ b/plugins/python/webgui/js/pythonMenu.js @@ -8,5 +8,99 @@ require([ 'codecompass/urlHandler', 'codecompass/viewHandler'], function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, urlHandler, viewHandler) { + model.addService('pythonservice', 'PythonService', LanguageServiceClient); -}); \ No newline at end of file + + var getdefintion = { + id : 'python-text-getdefintion', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Jump to definition', + accelKey : 'ctrl - click', + onClick : function () { + if (!nodeInfo || !fileInfo) + return; + + var languageService = model.getLanguageService(fileInfo.type); + astHelper.jumpToDef(nodeInfo.id, model.pythonservice); + + if (window.gtag) { + window.gtag ('event', 'jump_to_def', { + 'event_category' : urlHandler.getState('wsid'), + 'event_label' : urlHandler.getFileInfo().name + + ': ' + + nodeInfo.astNodeValue + }); + } + } + }); + } + }; + + viewHandler.registerModule(getdefintion, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var infoTree = { + id : 'python-text-infotree', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Info Tree', + onClick : function () { + if (!nodeInfo || !fileInfo) + return; + + topic.publish('codecompass/infotree', { + fileType : fileInfo.type, + elementInfo : nodeInfo + }); + + if (window.gtag) { + window.gtag ('event', 'info_tree', { + 'event_category' : urlHandler.getState('wsid'), + 'event_label' : urlHandler.getFileInfo().name + + ': ' + + nodeInfo.astNodeValue + }); + } + } + }); + } + }; + + viewHandler.registerModule(infoTree, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var infobox = { + id : 'python-text-infobox', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Documentation', + onClick : function () { + topic.publish('codecompass/documentation', { + fileType : fileInfo.type, + elementInfo : nodeInfo + }); + + if (window.gtag) { + window.gtag ('event', 'documentation', { + 'event_category' : urlHandler.getState('wsid'), + 'event_label' : urlHandler.getFileInfo().name + + ': ' + + nodeInfo.astNodeValue + }); + } + } + }); + } + }; + + viewHandler.registerModule(infobox, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + +}); From 9752433c7cb9b8722ac5015b48a09a8be85838c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 27 Nov 2023 20:40:12 +0100 Subject: [PATCH 007/156] pythonplugin basic parser --- plugins/python/parser/CMakeLists.txt | 15 ++++++ .../include/pythonparser/pythonparser.h | 6 +++ plugins/python/parser/src/pythonparser.cpp | 48 ++++++++++++++----- plugins/python/service/src/pythonservice.cpp | 20 ++++++++ 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index 55c20f20a..845f91af7 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -1,12 +1,27 @@ +find_package(Python3 REQUIRED COMPONENTS Interpreter Development) +find_package(Boost REQUIRED COMPONENTS python) + include_directories( include ${PROJECT_SOURCE_DIR}/model/include ${PROJECT_SOURCE_DIR}/util/include ${PROJECT_SOURCE_DIR}/parser/include) +include_directories(SYSTEM + ${Boost_INCLUDE_DIRS} + ${Python3_INCLUDE_DIRS}) + add_library(pythonparser SHARED src/pythonparser.cpp) +target_link_libraries(pythonparser + model + ${Boost_LIBRARIES} + ${Python3_LIBRARIES}) + target_compile_options(pythonparser PUBLIC -Wno-unknown-pragmas) install(TARGETS pythonparser DESTINATION ${INSTALL_PARSER_DIR}) +install( + DIRECTORY ${PLUGIN_DIR}/parser/pyparser/ + DESTINATION ${INSTALL_PARSER_DIR}/pyparser/) diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index 8ae0595a1..d43719541 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -3,12 +3,17 @@ #include #include +#include +#include +#include namespace cc { namespace parser { +namespace python = boost::python; + class PythonParser : public AbstractParser { public: @@ -17,6 +22,7 @@ class PythonParser : public AbstractParser virtual bool parse() override; private: bool accept(const std::string& path_); + python::object m_py_module; }; } // parser diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index dc37d7400..ee1e763ef 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -1,9 +1,6 @@ #include - #include - #include - #include namespace cc @@ -13,21 +10,52 @@ namespace parser PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) { + // Init Python Interpreter + std::string py_parser_dir = _ctx.compassRoot + "/lib/parserplugin/pyparser/"; + LOG(info) << "py_parser_dir: " << py_parser_dir; + setenv("PYTHONPATH", py_parser_dir.c_str(), 1); + + Py_Initialize(); + + // Init PyService module + try { + m_py_module = python::import("parser"); + + // DEBUG + m_py_module.attr("hello")(); + }catch (const python::error_already_set&) + { + PyErr_Print(); + } + } bool PythonParser::accept(const std::string& path_) { std::string ext = boost::filesystem::extension(path_); - return ext == ".dummy"; + return ext == ".py"; } bool PythonParser::parse() -{ +{ for(std::string path : _ctx.options["input"].as>()) { - if(accept(path)) + if(boost::filesystem::is_directory(path)) { - LOG(info) << "DummyParser parse path: " << path; + util::iterateDirectoryRecursive(path, [this](const std::string& currPath_) + { + if (boost::filesystem::is_regular_file(currPath_) && accept(currPath_)) + { + LOG(info) << "PythonParser parse path: " << currPath_; + + model::FilePtr pyfile = _ctx.srcMgr.getFile(currPath_); + pyfile->parseStatus = model::File::ParseStatus::PSFullyParsed; + pyfile->type = "PY"; + _ctx.srcMgr.updateFile(*pyfile); + } + + return true; + }); } } return true; @@ -45,10 +73,6 @@ extern "C" { boost::program_options::options_description description("Python Plugin"); - description.add_options() - ("dummy-arg", po::value()->default_value("Dummy arg"), - "This argument will be used by the dummy parser."); - return description; } @@ -60,4 +84,4 @@ extern "C" #pragma clang diagnostic pop } // parser -} // cc +} // cc \ No newline at end of file diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 605e9ed1a..c490b9756 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -26,6 +26,7 @@ void PythonServiceHandler::getAstNodeInfo( AstNodeInfo& return_, const core::AstNodeId& astNodeId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -33,6 +34,7 @@ void PythonServiceHandler::getAstNodeInfoByPosition( AstNodeInfo& return_, const core::FilePosition& fpos_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -40,6 +42,7 @@ void PythonServiceHandler::getSourceText( std::string& return_, const core::AstNodeId& astNodeId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -47,6 +50,7 @@ void PythonServiceHandler::getDocumentation( std::string& return_, const core::AstNodeId& astNodeId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -54,6 +58,7 @@ void PythonServiceHandler::getProperties( std::map& return_, const core::AstNodeId& astNodeId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -61,6 +66,7 @@ void PythonServiceHandler::getDiagramTypes( std::map& return_, const core::AstNodeId& astNodeId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -69,6 +75,7 @@ void PythonServiceHandler::getDiagram( const core::AstNodeId& astNodeId_, const std::int32_t diagramId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -76,6 +83,7 @@ void PythonServiceHandler::getDiagramLegend( std::string& return_, const std::int32_t diagramId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -83,6 +91,7 @@ void PythonServiceHandler::getFileDiagramTypes( std::map& return_, const core::FileId& fileId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -91,6 +100,7 @@ void PythonServiceHandler::getFileDiagram( const core::FileId& fileId_, const int32_t diagramId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -98,6 +108,7 @@ void PythonServiceHandler::getFileDiagramLegend( std::string& return_, const std::int32_t diagramId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -105,6 +116,7 @@ void PythonServiceHandler::getReferenceTypes( std::map& return_, const core::AstNodeId& astNodeId) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -114,6 +126,7 @@ void PythonServiceHandler::getReferences( const std::int32_t referenceId_, const std::vector& tags_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -121,6 +134,7 @@ std::int32_t PythonServiceHandler::getReferenceCount( const core::AstNodeId& astNodeId_, const std::int32_t referenceId_) { + LOG(info) << "[PYSERVICE] " << __func__; return 0; } @@ -131,6 +145,7 @@ void PythonServiceHandler::getReferencesInFile( const core::FileId& fileId_, const std::vector& tags_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -141,6 +156,7 @@ void PythonServiceHandler::getReferencesPage( const std::int32_t pageSize_, const std::int32_t pageNo_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -148,6 +164,7 @@ void PythonServiceHandler::getFileReferenceTypes( std::map& return_, const core::FileId& fileId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -156,6 +173,7 @@ void PythonServiceHandler::getFileReferences( const core::FileId& fileId_, const std::int32_t referenceId_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } @@ -163,6 +181,7 @@ std::int32_t PythonServiceHandler::getFileReferenceCount( const core::FileId& fileId_, const std::int32_t referenceId_) { + LOG(info) << "[PYSERVICE] " << __func__; return 0; } @@ -170,6 +189,7 @@ void PythonServiceHandler::getSyntaxHighlight( std::vector& return_, const core::FileRange& range_) { + LOG(info) << "[PYSERVICE] " << __func__; return; } From 68ca707ec5ef5a892b7459f1f9e2ed45a1d83629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 30 Nov 2023 16:58:27 +0100 Subject: [PATCH 008/156] python parser --- plugins/python/parser/src/pythonparser.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index ee1e763ef..6b6f708e1 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -20,9 +20,6 @@ PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) // Init PyService module try { m_py_module = python::import("parser"); - - // DEBUG - m_py_module.attr("hello")(); }catch (const python::error_already_set&) { PyErr_Print(); @@ -48,10 +45,21 @@ bool PythonParser::parse() { LOG(info) << "PythonParser parse path: " << currPath_; - model::FilePtr pyfile = _ctx.srcMgr.getFile(currPath_); - pyfile->parseStatus = model::File::ParseStatus::PSFullyParsed; - pyfile->type = "PY"; - _ctx.srcMgr.updateFile(*pyfile); + try { + bool parsed = python::extract(m_py_module.attr("parse")(currPath_)); + + if(parsed) + { + model::FilePtr pyfile = _ctx.srcMgr.getFile(currPath_); + pyfile->parseStatus = model::File::ParseStatus::PSFullyParsed; + pyfile->type = "PY"; + _ctx.srcMgr.updateFile(*pyfile); + } + + }catch (const python::error_already_set&) + { + PyErr_Print(); + } } return true; From 4bb2349b311917c4881484793f42fdada9d38b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 4 Dec 2023 18:11:54 +0100 Subject: [PATCH 009/156] pythonplugin model --- plugins/python/CMakeLists.txt | 3 ++- plugins/python/model/CMakeLists.txt | 10 ++++++++++ plugins/python/model/include/model/pyname.h | 21 +++++++++++++++++++++ plugins/python/parser/CMakeLists.txt | 3 ++- plugins/python/parser/pyparser/parser.py | 0 5 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 plugins/python/model/CMakeLists.txt create mode 100644 plugins/python/model/include/model/pyname.h create mode 100644 plugins/python/parser/pyparser/parser.py diff --git a/plugins/python/CMakeLists.txt b/plugins/python/CMakeLists.txt index d7aed06be..5aa4665f9 100644 --- a/plugins/python/CMakeLists.txt +++ b/plugins/python/CMakeLists.txt @@ -1,4 +1,5 @@ +add_subdirectory(model) add_subdirectory(parser) -#add_subdirectory(test) add_subdirectory(service) +#add_subdirectory(test) install_webplugin(webgui) diff --git a/plugins/python/model/CMakeLists.txt b/plugins/python/model/CMakeLists.txt new file mode 100644 index 000000000..87e661e87 --- /dev/null +++ b/plugins/python/model/CMakeLists.txt @@ -0,0 +1,10 @@ +set(ODB_SOURCES + include/model/pyname.h +) + +generate_odb_files("${ODB_SOURCES}") + +add_odb_library(pythonmodel ${ODB_CXX_SOURCES}) +target_link_libraries(pythonmodel model) + +install_sql() diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h new file mode 100644 index 000000000..cf9513af2 --- /dev/null +++ b/plugins/python/model/include/model/pyname.h @@ -0,0 +1,21 @@ +#ifndef CC_MODEL_PYNAME_H +#define CC_MODEL_PYNAME_H + +#include +#include + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PYName +{ + #pragma db id + std::uint64_t id = 0; +}; +} +} + +#endif \ No newline at end of file diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index 845f91af7..9bf0b51ae 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -5,7 +5,8 @@ include_directories( include ${PROJECT_SOURCE_DIR}/model/include ${PROJECT_SOURCE_DIR}/util/include - ${PROJECT_SOURCE_DIR}/parser/include) + ${PROJECT_SOURCE_DIR}/parser/include + ${PLUGIN_DIR}/model/include) include_directories(SYSTEM ${Boost_INCLUDE_DIRS} diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py new file mode 100644 index 000000000..e69de29bb From a8996746ac742f954fa5e68a65ead8a7e3733ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 13 Feb 2024 16:06:47 +0100 Subject: [PATCH 010/156] PYName model --- plugins/python/parser/CMakeLists.txt | 1 + plugins/python/parser/include/pythonparser/pythonparser.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index 9bf0b51ae..b1d6bc7be 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -17,6 +17,7 @@ add_library(pythonparser SHARED target_link_libraries(pythonparser model + pythonmodel ${Boost_LIBRARIES} ${Python3_LIBRARIES}) diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index d43719541..3c5c8fdb3 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -6,7 +6,8 @@ #include #include #include - +#include +#include namespace cc { namespace parser From 9be66d1f2b179a68da77e0aaeb5afa9278dd1df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 13 Feb 2024 16:17:37 +0100 Subject: [PATCH 011/156] PythonParser parse function --- plugins/python/parser/pyparser/parser.py | 3 +++ plugins/python/parser/src/pythonparser.cpp | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index e69de29bb..c76f1273d 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -0,0 +1,3 @@ +def parse(path): + print(f"[PythonParser] Parsing: {path}") + return True diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 6b6f708e1..c290ae1ef 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -43,8 +43,6 @@ bool PythonParser::parse() { if (boost::filesystem::is_regular_file(currPath_) && accept(currPath_)) { - LOG(info) << "PythonParser parse path: " << currPath_; - try { bool parsed = python::extract(m_py_module.attr("parse")(currPath_)); From 2f0b3ce2c6697ca218c638fde8b2d4e77fac95a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 14 Feb 2024 16:47:56 +0100 Subject: [PATCH 012/156] PythonParser parsing a file --- plugins/python/model/include/model/pyname.h | 4 +++ plugins/python/parser/pyparser/parser.py | 35 ++++++++++++++++++++- plugins/python/parser/src/pythonparser.cpp | 14 +++++++-- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index cf9513af2..a134aeb45 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -14,6 +14,10 @@ struct PYName { #pragma db id std::uint64_t id = 0; + + std::uint64_t line; + std::uint64_t column; + std::uint64_t refID; }; } } diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index c76f1273d..6b0bd56c3 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -1,3 +1,36 @@ +import jedi +from hashlib import sha1 + def parse(path): print(f"[PythonParser] Parsing: {path}") - return True + + with open(path) as f: + source = f.read() + script = jedi.Script(source, path=path) + + names_arr = [] + + for x in script.get_names(definitions = True, references = True): + name = {} + + name["module_name"] = x.module_name + name["module_path"] = x.module_path + name["full_name"] = x.full_name + name["line_start"] = x.get_definition_start_position()[0] + name["line_end"] = x.get_definition_end_position()[0] + name["line"] = x.line + name["column"] = x.column + name["column_start"] = x.get_definition_start_position()[1] + name["column_end"] = x.get_definition_end_position()[1] + name["type"] = x.type + name["definition"] = x.is_definition() + + names_arr.append(name) + + return names_arr + +def hashName(file_path, line, start_column, end_column): + s = f"{file_path}|{line}|{start_column}|{end_column}".encode("utf-8") + hash = str(sha1(s).hexdigest()) + return hash + diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index c290ae1ef..e12bcd7cc 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace cc { @@ -17,7 +18,7 @@ PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) Py_Initialize(); - // Init PyService module + // Init PyParser module try { m_py_module = python::import("parser"); }catch (const python::error_already_set&) @@ -44,9 +45,16 @@ bool PythonParser::parse() if (boost::filesystem::is_regular_file(currPath_) && accept(currPath_)) { try { - bool parsed = python::extract(m_py_module.attr("parse")(currPath_)); - if(parsed) + python::object refs = m_py_module.attr("parse")(currPath_); + const int len = python::len(refs); + for (int i = 0; i < len; i++) + { + python::object node = refs[i]; + std::string full_name = python::extract(node["full_name"]); + } + + if(len > 0) { model::FilePtr pyfile = _ctx.srcMgr.getFile(currPath_); pyfile->parseStatus = model::File::ParseStatus::PSFullyParsed; From 2b07a9b4b6f2298b86e665145d9654f95deee267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 14 Feb 2024 21:59:47 +0100 Subject: [PATCH 013/156] PYName hash --- plugins/python/parser/pyparser/parser.py | 7 ++++--- plugins/python/parser/src/pythonparser.cpp | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 6b0bd56c3..df0f2cc28 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -13,6 +13,7 @@ def parse(path): for x in script.get_names(definitions = True, references = True): name = {} + name["hash"] = hashName(x) name["module_name"] = x.module_name name["module_path"] = x.module_path name["full_name"] = x.full_name @@ -29,8 +30,8 @@ def parse(path): return names_arr -def hashName(file_path, line, start_column, end_column): - s = f"{file_path}|{line}|{start_column}|{end_column}".encode("utf-8") - hash = str(sha1(s).hexdigest()) +def hashName(name): + s = f"{name.module_path}|{name.line}|{name.column}".encode("utf-8") + hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff return hash diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index e12bcd7cc..e39df4f10 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -51,7 +51,7 @@ bool PythonParser::parse() for (int i = 0; i < len; i++) { python::object node = refs[i]; - std::string full_name = python::extract(node["full_name"]); + uint64_t hash = python::extract(node["hash"]); } if(len > 0) From bbb03288530621087f3063fc61132e8805dd8524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 16 Feb 2024 17:41:44 +0100 Subject: [PATCH 014/156] Inserting PYName into database --- plugins/python/model/include/model/pyname.h | 16 ++- .../include/pythonparser/pythonparser.h | 6 + plugins/python/parser/pyparser/parser.py | 87 ++++++++++--- plugins/python/parser/src/pythonparser.cpp | 120 ++++++++++++++---- 4 files changed, 184 insertions(+), 45 deletions(-) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index a134aeb45..1628ed03a 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -2,6 +2,7 @@ #define CC_MODEL_PYNAME_H #include +#include #include namespace cc @@ -12,12 +13,25 @@ namespace model #pragma db object struct PYName { + enum class PYNameType + { + Module, Class, Instance, Function, Param, Path, Keyword, Property, Statement, Unknown + }; + #pragma db id std::uint64_t id = 0; + std::uint64_t refid; + bool is_definition = false; + bool is_builtin = false; + std::string full_name; + PYNameType type; std::uint64_t line; std::uint64_t column; - std::uint64_t refID; + std::uint64_t line_start; + std::uint64_t line_end; + std::uint64_t column_start; + std::uint64_t column_end; }; } } diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index 3c5c8fdb3..e345c1b0d 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -1,6 +1,8 @@ #ifndef CC_PARSER_PYTHONPARSER_H #define CC_PARSER_PYTHONPARSER_H +#include +#include #include #include #include @@ -15,6 +17,8 @@ namespace parser namespace python = boost::python; +typedef std::unordered_map PYNameMap; + class PythonParser : public AbstractParser { public: @@ -23,6 +27,8 @@ class PythonParser : public AbstractParser virtual bool parse() override; private: bool accept(const std::string& path_); + void parseFile(const std::string& path_, PYNameMap& map); + model::PYName::PYNameType getPYNameType(const std::string& str); python::object m_py_module; }; diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index df0f2cc28..90c88104e 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -1,34 +1,79 @@ import jedi from hashlib import sha1 +def log(msg): + print(f"[PythonParser] {msg}") + def parse(path): - print(f"[PythonParser] Parsing: {path}") + log(f"[PythonParser] Parsing: {path}") + + result = { + "status": "full", + "nodes": [] + } with open(path) as f: source = f.read() script = jedi.Script(source, path=path) - - names_arr = [] - for x in script.get_names(definitions = True, references = True): - name = {} + nodes = {} + + for x in script.get_names(references = True): + defs = x.goto(follow_imports=True) + + if len(defs) > 0: + refid = min(list(map(lambda x : hashName(x), defs))) + else: + result["status"] = "partial" + log(f"No definition found for {x.full_name}") + log(f"{x.full_name}: file = {x.module_path} line = {x.line} column = {x.column}") + refid = hashName(x) - name["hash"] = hashName(x) - name["module_name"] = x.module_name - name["module_path"] = x.module_path - name["full_name"] = x.full_name - name["line_start"] = x.get_definition_start_position()[0] - name["line_end"] = x.get_definition_end_position()[0] - name["line"] = x.line - name["column"] = x.column - name["column_start"] = x.get_definition_start_position()[1] - name["column_end"] = x.get_definition_end_position()[1] - name["type"] = x.type - name["definition"] = x.is_definition() - - names_arr.append(name) - - return names_arr + putInMap(nodes, getNodeInfo(x, refid)) + + for d in defs: + putInMap(nodes, getNodeInfo(d, refid)) + + result["nodes"] = list(nodes.values()) + + if len(result["nodes"]) == 0: + result["status"] = "none" + + return result + +def getNodeInfo(name, refid): + node = {} + node["id"] = hashName(name) + node["refid"] = refid + node["module_name"] = name.module_name + node["module_path"] = name.module_path + node["full_name"] = name.full_name if name.full_name else "" + + node["line"] = 0 + node["column"] = 0 + node["line_start"] = 0 + node["line_end"] = 0 + node["column_start"] = 0 + node["column_end"] = 0 + + if name.line and name.column: + node["line"] = name.line + node["column"] = name.column + + if name.get_definition_start_position(): + node["line_start"] = name.get_definition_start_position()[0] + node["line_end"] = name.get_definition_end_position()[0] + node["column_start"] = name.get_definition_start_position()[1] + node["column_end"] = name.get_definition_end_position()[1] + + node["type"] = name.type + node["is_definition"] = name.is_definition() + node["is_builtin"] = name.in_builtin_module() + + return node + +def putInMap(hashmap, node): + hashmap[node["id"]] = node def hashName(name): s = f"{name.module_path}|{name.line}|{name.column}".encode("utf-8") diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index e39df4f10..7f5644451 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -34,44 +34,118 @@ bool PythonParser::accept(const std::string& path_) return ext == ".py"; } +model::PYName::PYNameType PythonParser::getPYNameType(const std::string& str) +{ + static std::unordered_map const table = { + {"class",model::PYName::PYNameType::Class}, + {"function",model::PYName::PYNameType::Function}, + {"instance",model::PYName::PYNameType::Instance}, + {"keyword",model::PYName::PYNameType::Keyword}, + {"module",model::PYName::PYNameType::Module}, + {"param",model::PYName::PYNameType::Param}, + {"path",model::PYName::PYNameType::Path}, + {"property",model::PYName::PYNameType::Property}, + {"statement",model::PYName::PYNameType::Statement}, + {"unknown",model::PYName::PYNameType::Unknown} + }; + + auto it = table.find(str); + if (it != table.end()) { + return it->second; + } else { + return model::PYName::PYNameType::Unknown; + } +} + +void PythonParser::parseFile(const std::string& path_, PYNameMap& map) +{ + try { + + python::object result = m_py_module.attr("parse")(path_); + python::object nodes = result["nodes"]; + std::string status = python::extract(result["status"]); + + const int len = python::len(nodes); + for (int i = 0; i < len; i++) + { + python::object node = nodes[i]; + + model::PYName pyname; + pyname.id = python::extract(node["id"]); + pyname.refid = python::extract(node["refid"]); + pyname.full_name = python::extract(node["full_name"]); + pyname.is_definition = python::extract(node["is_definition"]); + pyname.is_builtin = python::extract(node["is_builtin"]); + pyname.line = python::extract(node["line"]); + pyname.column = python::extract(node["column"]); + pyname.line_start = python::extract(node["line_start"]); + pyname.line_end = python::extract(node["line_end"]); + pyname.column_start = python::extract(node["column_start"]); + pyname.column_end = python::extract(node["column_end"]); + + // PYNameType + std::string type_str = python::extract(node["type"]); + pyname.type = PythonParser::getPYNameType(type_str); + + // Put in map + if(map.find(pyname.id) == map.end()) + { + map[pyname.id] = pyname; + } + } + + if(status != "none") + { + model::FilePtr pyfile = _ctx.srcMgr.getFile(path_); + + if(status == "full") + { + pyfile->parseStatus = model::File::ParseStatus::PSFullyParsed; + }else if (status == "partial") + { + pyfile->parseStatus = model::File::ParseStatus::PSPartiallyParsed; + } + + pyfile->type = "PY"; + _ctx.srcMgr.updateFile(*pyfile); + } + + }catch (const python::error_already_set&) + { + PyErr_Print(); + } +} + bool PythonParser::parse() { + PYNameMap map; + for(std::string path : _ctx.options["input"].as>()) { if(boost::filesystem::is_directory(path)) { - util::iterateDirectoryRecursive(path, [this](const std::string& currPath_) + util::iterateDirectoryRecursive(path, [&](const std::string& currPath_) { if (boost::filesystem::is_regular_file(currPath_) && accept(currPath_)) { - try { - - python::object refs = m_py_module.attr("parse")(currPath_); - const int len = python::len(refs); - for (int i = 0; i < len; i++) - { - python::object node = refs[i]; - uint64_t hash = python::extract(node["hash"]); - } - - if(len > 0) - { - model::FilePtr pyfile = _ctx.srcMgr.getFile(currPath_); - pyfile->parseStatus = model::File::ParseStatus::PSFullyParsed; - pyfile->type = "PY"; - _ctx.srcMgr.updateFile(*pyfile); - } - - }catch (const python::error_already_set&) - { - PyErr_Print(); - } + PythonParser::parseFile(currPath_, map); } return true; }); } } + + // Insert into database + for(const auto& [key, value] : map) + { + LOG(debug) << "Inserting PYName " << value.id; + cc::util::OdbTransaction {_ctx.db} ([&] + { + _ctx.db->persist(value); + }); + } + return true; } From 50c88af3697907c1e3910b4546d7bb9d24918315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 16 Feb 2024 20:06:37 +0100 Subject: [PATCH 015/156] PYName added file_id --- plugins/python/model/include/model/pyname.h | 3 ++- plugins/python/parser/pyparser/parser.py | 15 ++++++++++++++- plugins/python/parser/src/pythonparser.cpp | 3 ++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index 1628ed03a..4f47e46be 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -21,7 +21,7 @@ struct PYName #pragma db id std::uint64_t id = 0; - std::uint64_t refid; + std::uint64_t ref_id; bool is_definition = false; bool is_builtin = false; std::string full_name; @@ -32,6 +32,7 @@ struct PYName std::uint64_t line_end; std::uint64_t column_start; std::uint64_t column_end; + std::uint64_t file_id; }; } } diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 90c88104e..45a3169f9 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -44,7 +44,7 @@ def parse(path): def getNodeInfo(name, refid): node = {} node["id"] = hashName(name) - node["refid"] = refid + node["ref_id"] = refid node["module_name"] = name.module_name node["module_path"] = name.module_path node["full_name"] = name.full_name if name.full_name else "" @@ -69,6 +69,7 @@ def getNodeInfo(name, refid): node["type"] = name.type node["is_definition"] = name.is_definition() node["is_builtin"] = name.in_builtin_module() + node["file_id"] = getFileId(name) return node @@ -80,3 +81,15 @@ def hashName(name): hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff return hash +def fnvHash(str): + hash = 14695981039346656037 + + for c in str: + hash ^= ord(c) + hash *= 1099511628211 + + # see: https://stackoverflow.com/questions/20766813/how-to-convert-signed-to-unsigned-integer-in-python + return hash & 0xffffffffffffffff + +def getFileId(name): + return fnvHash(str(name.module_path)) \ No newline at end of file diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 7f5644451..7463829d7 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -72,7 +72,7 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) model::PYName pyname; pyname.id = python::extract(node["id"]); - pyname.refid = python::extract(node["refid"]); + pyname.ref_id = python::extract(node["ref_id"]); pyname.full_name = python::extract(node["full_name"]); pyname.is_definition = python::extract(node["is_definition"]); pyname.is_builtin = python::extract(node["is_builtin"]); @@ -82,6 +82,7 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) pyname.line_end = python::extract(node["line_end"]); pyname.column_start = python::extract(node["column_start"]); pyname.column_end = python::extract(node["column_end"]); + pyname.file_id = python::extract(node["file_id"]); // PYNameType std::string type_str = python::extract(node["type"]); From 550370c4465d5c817c47f91a3281f561d2dd8e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 16 Feb 2024 22:24:04 +0100 Subject: [PATCH 016/156] PythonService queries PYName from database --- plugins/python/model/include/model/pyname.h | 11 ++----- .../include/pythonparser/pythonparser.h | 1 - plugins/python/parser/pyparser/parser.py | 12 +++---- plugins/python/parser/src/pythonparser.cpp | 32 ++----------------- plugins/python/service/CMakeLists.txt | 5 ++- .../service/include/service/pythonservice.h | 3 ++ plugins/python/service/src/pythonservice.cpp | 17 ++++++++++ 7 files changed, 34 insertions(+), 47 deletions(-) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index 4f47e46be..d3c247d96 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -13,11 +13,6 @@ namespace model #pragma db object struct PYName { - enum class PYNameType - { - Module, Class, Instance, Function, Param, Path, Keyword, Property, Statement, Unknown - }; - #pragma db id std::uint64_t id = 0; @@ -25,15 +20,15 @@ struct PYName bool is_definition = false; bool is_builtin = false; std::string full_name; - PYNameType type; - std::uint64_t line; - std::uint64_t column; + std::string value; + std::string type; std::uint64_t line_start; std::uint64_t line_end; std::uint64_t column_start; std::uint64_t column_end; std::uint64_t file_id; }; + } } diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index e345c1b0d..49b1e3ce4 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -28,7 +28,6 @@ class PythonParser : public AbstractParser private: bool accept(const std::string& path_); void parseFile(const std::string& path_, PYNameMap& map); - model::PYName::PYNameType getPYNameType(const std::string& str); python::object m_py_module; }; diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 45a3169f9..571704fa5 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -49,22 +49,18 @@ def getNodeInfo(name, refid): node["module_path"] = name.module_path node["full_name"] = name.full_name if name.full_name else "" - node["line"] = 0 - node["column"] = 0 node["line_start"] = 0 node["line_end"] = 0 node["column_start"] = 0 node["column_end"] = 0 - - if name.line and name.column: - node["line"] = name.line - node["column"] = name.column + node["value"] = "" if name.get_definition_start_position(): node["line_start"] = name.get_definition_start_position()[0] node["line_end"] = name.get_definition_end_position()[0] - node["column_start"] = name.get_definition_start_position()[1] - node["column_end"] = name.get_definition_end_position()[1] + node["column_start"] = name.get_definition_start_position()[1] + 1 + node["column_end"] = name.get_definition_end_position()[1] + 1 + node["value"] = name.get_line_code() node["type"] = name.type node["is_definition"] = name.is_definition() diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 7463829d7..0ea0a043e 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -34,33 +34,11 @@ bool PythonParser::accept(const std::string& path_) return ext == ".py"; } -model::PYName::PYNameType PythonParser::getPYNameType(const std::string& str) -{ - static std::unordered_map const table = { - {"class",model::PYName::PYNameType::Class}, - {"function",model::PYName::PYNameType::Function}, - {"instance",model::PYName::PYNameType::Instance}, - {"keyword",model::PYName::PYNameType::Keyword}, - {"module",model::PYName::PYNameType::Module}, - {"param",model::PYName::PYNameType::Param}, - {"path",model::PYName::PYNameType::Path}, - {"property",model::PYName::PYNameType::Property}, - {"statement",model::PYName::PYNameType::Statement}, - {"unknown",model::PYName::PYNameType::Unknown} - }; - - auto it = table.find(str); - if (it != table.end()) { - return it->second; - } else { - return model::PYName::PYNameType::Unknown; - } -} - void PythonParser::parseFile(const std::string& path_, PYNameMap& map) { try { + // Call PythonParser parse(path) python::object result = m_py_module.attr("parse")(path_); python::object nodes = result["nodes"]; std::string status = python::extract(result["status"]); @@ -76,17 +54,13 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) pyname.full_name = python::extract(node["full_name"]); pyname.is_definition = python::extract(node["is_definition"]); pyname.is_builtin = python::extract(node["is_builtin"]); - pyname.line = python::extract(node["line"]); - pyname.column = python::extract(node["column"]); pyname.line_start = python::extract(node["line_start"]); pyname.line_end = python::extract(node["line_end"]); pyname.column_start = python::extract(node["column_start"]); pyname.column_end = python::extract(node["column_end"]); pyname.file_id = python::extract(node["file_id"]); - - // PYNameType - std::string type_str = python::extract(node["type"]); - pyname.type = PythonParser::getPYNameType(type_str); + pyname.value = python::extract(node["value"]); + pyname.type = python::extract(node["type"]); // Put in map if(map.find(pyname.id) == map.end()) diff --git a/plugins/python/service/CMakeLists.txt b/plugins/python/service/CMakeLists.txt index f755ba09c..e145da72d 100644 --- a/plugins/python/service/CMakeLists.txt +++ b/plugins/python/service/CMakeLists.txt @@ -5,7 +5,8 @@ include_directories( ${PROJECT_SOURCE_DIR}/webserver/include ${PROJECT_BINARY_DIR}/service/language/gen-cpp ${PROJECT_BINARY_DIR}/service/project/gen-cpp - ${PROJECT_SOURCE_DIR}/service/project/include) + ${PROJECT_SOURCE_DIR}/service/project/include + ${PLUGIN_DIR}/model/include) include_directories(SYSTEM ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}) @@ -18,6 +19,8 @@ target_compile_options(pythonservice PUBLIC -Wno-unknown-pragmas) target_link_libraries(pythonservice util + model + pythonmodel projectservice languagethrift ${THRIFT_LIBTHRIFT_LIBRARIES} diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 1040cfed9..59733a9b1 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -12,6 +12,9 @@ #include +#include +#include + namespace cc { namespace service diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index c490b9756..40dead522 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -35,6 +35,23 @@ void PythonServiceHandler::getAstNodeInfoByPosition( const core::FilePosition& fpos_) { LOG(info) << "[PYSERVICE] " << __func__; + _transaction([&]() { + auto nodes = _db->query( + odb::query::file_id == std::stoull(fpos_.file) && + odb::query::line_start == fpos_.pos.line && + odb::query::column_start <= fpos_.pos.column && + odb::query::column_end >= fpos_.pos.column + ); + + if(!nodes.empty()) + { + model::PYName pyname = *nodes.begin(); + return_.id = pyname.id; + return_.astNodeValue = pyname.value; + }else{ + LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos_.pos.line << " column = " << fpos_.pos.column << ")"; + } + }); return; } From 5a238e23b280b21796301e820d926463650a41e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 19 Feb 2024 11:29:56 +0100 Subject: [PATCH 017/156] Update PyName hash function --- plugins/python/parser/pyparser/parser.py | 35 +++++++++++++++--------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 571704fa5..8fbcc06b4 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -49,19 +49,9 @@ def getNodeInfo(name, refid): node["module_path"] = name.module_path node["full_name"] = name.full_name if name.full_name else "" - node["line_start"] = 0 - node["line_end"] = 0 - node["column_start"] = 0 - node["column_end"] = 0 - node["value"] = "" + pos = getNamePosInfo(name) + node.update(pos) # merge pos dictionary - if name.get_definition_start_position(): - node["line_start"] = name.get_definition_start_position()[0] - node["line_end"] = name.get_definition_end_position()[0] - node["column_start"] = name.get_definition_start_position()[1] + 1 - node["column_end"] = name.get_definition_end_position()[1] + 1 - node["value"] = name.get_line_code() - node["type"] = name.type node["is_definition"] = name.is_definition() node["is_builtin"] = name.in_builtin_module() @@ -69,11 +59,30 @@ def getNodeInfo(name, refid): return node +def getNamePosInfo(name): + pos = { + "line_start": 0, + "line_end": 0, + "column_start": 0, + "column_end": 0, + "value": "" + } + + if name.get_definition_start_position(): + pos["line_start"] = name.get_definition_start_position()[0] + pos["line_end"] = name.get_definition_end_position()[0] + pos["column_start"] = name.get_definition_start_position()[1] + 1 + pos["column_end"] = name.get_definition_end_position()[1] + 1 + pos["value"] = name.get_line_code() + + return pos + def putInMap(hashmap, node): hashmap[node["id"]] = node def hashName(name): - s = f"{name.module_path}|{name.line}|{name.column}".encode("utf-8") + pos = getNamePosInfo(name) + s = f"{name.module_path}|{pos['line_start']}|{pos['line_end']}|{pos['column_start']}|{pos['column_end']}".encode("utf-8") hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff return hash From 470490c0b3dd3cb29141b1be779503447134cd00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 20 Feb 2024 16:42:42 +0100 Subject: [PATCH 018/156] PythonService find references. --- plugins/python/parser/pyparser/parser.py | 4 +- plugins/python/service/src/pythonservice.cpp | 64 +++++++++++++++++++- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 8fbcc06b4..5bfd3627f 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -18,8 +18,8 @@ def parse(path): nodes = {} - for x in script.get_names(references = True): - defs = x.goto(follow_imports=True) + for x in script.get_names(references = True, all_scopes = True): + defs = x.goto(follow_imports = True) if len(defs) > 0: refid = min(list(map(lambda x : hashName(x), defs))) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 40dead522..cb594ae2a 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -46,8 +46,9 @@ void PythonServiceHandler::getAstNodeInfoByPosition( if(!nodes.empty()) { model::PYName pyname = *nodes.begin(); - return_.id = pyname.id; + return_.id = std::to_string(pyname.id); return_.astNodeValue = pyname.value; + return_.entityHash = 68; }else{ LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos_.pos.line << " column = " << fpos_.pos.column << ")"; } @@ -134,6 +135,8 @@ void PythonServiceHandler::getReferenceTypes( const core::AstNodeId& astNodeId) { LOG(info) << "[PYSERVICE] " << __func__; + return_.emplace("Definition", DEFINITION); + return_.emplace("Usage", USAGE); return; } @@ -144,6 +147,41 @@ void PythonServiceHandler::getReferences( const std::vector& tags_) { LOG(info) << "[PYSERVICE] " << __func__; + LOG(info) << "astNodeID: " << astNodeId_; + LOG(info) << "referenceId_: " << referenceId_; + + _transaction([&]() { + + std::uint64_t ref_id = _db->query_value(odb::query::id == std::stoull(astNodeId_)).ref_id; + + odb::result nodes; + switch (referenceId_) + { + case DEFINITION: + nodes = _db->query(odb::query::ref_id == ref_id && odb::query::is_definition == true); + break; + case USAGE: + nodes = _db->query(odb::query::ref_id == ref_id && odb::query::is_definition == false); + break; + } + + for(const model::PYName& pyname : nodes) + { + AstNodeInfo info; + info.id = pyname.id; + info.astNodeType = pyname.type; + info.range.file = std::to_string(pyname.file_id); + info.range.range.startpos.line = pyname.line_start; + info.range.range.startpos.column = pyname.column_start; + info.range.range.endpos.line = pyname.line_end; + info.range.range.endpos.column = pyname.column_end; + info.astNodeValue = pyname.value; + return_.push_back(info); + } + + return; + }); + return; } @@ -152,7 +190,27 @@ std::int32_t PythonServiceHandler::getReferenceCount( const std::int32_t referenceId_) { LOG(info) << "[PYSERVICE] " << __func__; - return 0; + LOG(info) << "astNodeID: " << astNodeId_; + LOG(info) << "referenceId_: " << referenceId_; + + std::int32_t ret = 0; + _transaction([&]() { + + std::uint64_t ref_id = _db->query_value(odb::query::id == std::stoull(astNodeId_)).ref_id; + + switch (referenceId_) + { + case DEFINITION: + ret = _db->query(odb::query::ref_id == ref_id && odb::query::is_definition == true).size(); + break; + case USAGE: + ret = _db->query(odb::query::ref_id == ref_id && odb::query::is_definition == false).size(); + break; + } + return; + }); + + return ret; } void PythonServiceHandler::getReferencesInFile( @@ -182,6 +240,8 @@ void PythonServiceHandler::getFileReferenceTypes( const core::FileId& fileId_) { LOG(info) << "[PYSERVICE] " << __func__; + return_["Definition"] = DEFINITION; + return_["Usage"] = USAGE; return; } From c46a43ebb8c8b7233b2890899c84d2c975c4fc1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 21 Feb 2024 23:34:27 +0100 Subject: [PATCH 019/156] Added support for Python virtual environments. --- plugins/python/parser/pyparser/parser.py | 24 ++++++++++++++++++---- plugins/python/parser/src/pythonparser.cpp | 12 ++++++++++- plugins/python/service/src/plugin.cpp | 5 ----- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 5bfd3627f..60b0aec4c 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -1,22 +1,38 @@ import jedi from hashlib import sha1 +config = { + "env": None, + "env_path": None +} + def log(msg): print(f"[PythonParser] {msg}") -def parse(path): - log(f"[PythonParser] Parsing: {path}") +def venv_config(venv_path): + try: + config["env"] = jedi.create_environment(venv_path) + config["env_path"] = venv_path + log(f"Using virtual environment: {venv_path}") + except: + log(f"Failed to use virtual environment: {venv_path}") +def parse(path): result = { - "status": "full", + "status": "none", "nodes": [] } with open(path) as f: + if config["env_path"] and path.startswith(config["env_path"]): + return result + + log(f"Parsing: {path}") source = f.read() - script = jedi.Script(source, path=path) + script = jedi.Script(source, path=path, environment=config["env"]) nodes = {} + result["status"] = "full" for x in script.get_names(references = True, all_scopes = True): defs = x.goto(follow_imports = True) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 0ea0a043e..7fd958dcb 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -13,7 +13,6 @@ PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) { // Init Python Interpreter std::string py_parser_dir = _ctx.compassRoot + "/lib/parserplugin/pyparser/"; - LOG(info) << "py_parser_dir: " << py_parser_dir; setenv("PYTHONPATH", py_parser_dir.c_str(), 1); Py_Initialize(); @@ -21,6 +20,14 @@ PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) // Init PyParser module try { m_py_module = python::import("parser"); + + // Set venv + if (_ctx.options.count("venv")) + { + std::string venv = _ctx.options["venv"].as(); + m_py_module.attr("venv_config")(venv); + } + }catch (const python::error_already_set&) { PyErr_Print(); @@ -135,6 +142,9 @@ extern "C" boost::program_options::options_description getOptions() { boost::program_options::options_description description("Python Plugin"); + description.add_options() + ("venv", po::value(), + "Set 'venv' to specify the project's Python virtual environment path."); return description; } diff --git a/plugins/python/service/src/plugin.cpp b/plugins/python/service/src/plugin.cpp index 0826da989..424c8f947 100644 --- a/plugins/python/service/src/plugin.cpp +++ b/plugins/python/service/src/plugin.cpp @@ -11,11 +11,6 @@ extern "C" namespace po = boost::program_options; po::options_description description("Python Plugin"); - - description.add_options() - ("dummy-result", po::value()->default_value("Dummy result"), - "This value will be returned by the dummy service."); - return description; } From 1f4093f71c5b33b20c58af5075b79c2a03c6c12d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 22 Feb 2024 12:02:45 +0100 Subject: [PATCH 020/156] Log parse results. --- plugins/python/parser/include/pythonparser/pythonparser.h | 6 ++++++ plugins/python/parser/pyparser/parser.py | 2 +- plugins/python/parser/src/pythonparser.cpp | 8 ++++++++ plugins/python/service/src/pythonservice.cpp | 2 -- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index 49b1e3ce4..622435a39 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -26,9 +26,15 @@ class PythonParser : public AbstractParser virtual ~PythonParser(); virtual bool parse() override; private: + struct ParseResult { + std::uint32_t partial; + std::uint32_t full; + }; + bool accept(const std::string& path_); void parseFile(const std::string& path_, PYNameMap& map); python::object m_py_module; + ParseResult m_parse_result; }; } // parser diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 60b0aec4c..e09fcfb86 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -35,7 +35,7 @@ def parse(path): result["status"] = "full" for x in script.get_names(references = True, all_scopes = True): - defs = x.goto(follow_imports = True) + defs = x.goto(follow_imports = True, follow_builtin_imports = True) if len(defs) > 0: refid = min(list(map(lambda x : hashName(x), defs))) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 7fd958dcb..bb09bea2b 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -82,9 +82,11 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) if(status == "full") { + m_parse_result.full++; pyfile->parseStatus = model::File::ParseStatus::PSFullyParsed; }else if (status == "partial") { + m_parse_result.partial++; pyfile->parseStatus = model::File::ParseStatus::PSPartiallyParsed; } @@ -101,6 +103,8 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) bool PythonParser::parse() { PYNameMap map; + m_parse_result.full = 0; + m_parse_result.partial = 0; for(std::string path : _ctx.options["input"].as>()) { @@ -128,6 +132,10 @@ bool PythonParser::parse() }); } + LOG(info) << "[PythonParser] Parsing finished!"; + LOG(info) << "[PythonParser] Fully parsed files: " << m_parse_result.full; + LOG(info) << "[PythonParser] Partially parsed files: " << m_parse_result.partial; + return true; } diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index cb594ae2a..7828eb94a 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -148,7 +148,6 @@ void PythonServiceHandler::getReferences( { LOG(info) << "[PYSERVICE] " << __func__; LOG(info) << "astNodeID: " << astNodeId_; - LOG(info) << "referenceId_: " << referenceId_; _transaction([&]() { @@ -191,7 +190,6 @@ std::int32_t PythonServiceHandler::getReferenceCount( { LOG(info) << "[PYSERVICE] " << __func__; LOG(info) << "astNodeID: " << astNodeId_; - LOG(info) << "referenceId_: " << referenceId_; std::int32_t ret = 0; _transaction([&]() { From a5786843dccbecec994022140f7d726dfdd982dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 22 Feb 2024 15:07:29 +0100 Subject: [PATCH 021/156] PythonParser prepare input --- .../include/pythonparser/pythonparser.h | 1 + plugins/python/parser/pyparser/parser.py | 20 +++++++++++------- plugins/python/parser/src/pythonparser.cpp | 21 +++++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index 622435a39..185657348 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -35,6 +35,7 @@ class PythonParser : public AbstractParser void parseFile(const std::string& path_, PYNameMap& map); python::object m_py_module; ParseResult m_parse_result; + void prepareInput(const std::string& root_path); }; } // parser diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index e09fcfb86..445455492 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -2,18 +2,22 @@ from hashlib import sha1 config = { - "env": None, - "env_path": None + "venv_path": None, + "project": None } def log(msg): print(f"[PythonParser] {msg}") -def venv_config(venv_path): +def project_config(root_path, venv_path = None): try: - config["env"] = jedi.create_environment(venv_path) - config["env_path"] = venv_path - log(f"Using virtual environment: {venv_path}") + if venv_path: + jedi.create_environment(venv_path) + config["venv_path"] = venv_path + log(f"Using virtual environment: {venv_path}") + + config["project"] = jedi.Project(path = root_path, environment_path = venv_path) + except: log(f"Failed to use virtual environment: {venv_path}") @@ -24,12 +28,12 @@ def parse(path): } with open(path) as f: - if config["env_path"] and path.startswith(config["env_path"]): + if config["venv_path"] and path.startswith(config["venv_path"]): return result log(f"Parsing: {path}") source = f.read() - script = jedi.Script(source, path=path, environment=config["env"]) + script = jedi.Script(source, path=path, project=config["project"]) nodes = {} result["status"] = "full" diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index bb09bea2b..bbd076d3b 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -35,6 +35,25 @@ PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) } +void PythonParser::prepareInput(const std::string& root_path) +{ + try { + if (_ctx.options.count("venv")) + { + std::string venv = _ctx.options["venv"].as(); + m_py_module.attr("project_config")(root_path, venv); + } + else + { + m_py_module.attr("project_config")(root_path); + } + + }catch (const python::error_already_set&) + { + PyErr_Print(); + } +} + bool PythonParser::accept(const std::string& path_) { std::string ext = boost::filesystem::extension(path_); @@ -108,6 +127,8 @@ bool PythonParser::parse() for(std::string path : _ctx.options["input"].as>()) { + PythonParser::prepareInput(path); + if(boost::filesystem::is_directory(path)) { util::iterateDirectoryRecursive(path, [&](const std::string& currPath_) From 02518b89826326e553e01e05e9b4715013a3c25d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 23 Feb 2024 10:42:29 +0100 Subject: [PATCH 022/156] PythonParser adding more parse statistics. --- plugins/python/parser/src/pythonparser.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index bbd076d3b..ed6656fc9 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -153,9 +153,10 @@ bool PythonParser::parse() }); } - LOG(info) << "[PythonParser] Parsing finished!"; - LOG(info) << "[PythonParser] Fully parsed files: " << m_parse_result.full; - LOG(info) << "[PythonParser] Partially parsed files: " << m_parse_result.partial; + LOG(info) << "[pythonparser] Parsing finished!"; + LOG(info) << "[pythonparser] Inserted PYName: " << map.size(); + LOG(info) << "[pythonparser] Fully parsed files: " << m_parse_result.full; + LOG(info) << "[pythonparser] Partially parsed files: " << m_parse_result.partial; return true; } From eb2a4cf8e14a47a57f8e3d8c28c210dfbcf43615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 23 Feb 2024 10:47:10 +0100 Subject: [PATCH 023/156] PythonParser removed venv_config --- plugins/python/parser/src/pythonparser.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index ed6656fc9..3d285fc2c 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -20,15 +20,8 @@ PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) // Init PyParser module try { m_py_module = python::import("parser"); - - // Set venv - if (_ctx.options.count("venv")) - { - std::string venv = _ctx.options["venv"].as(); - m_py_module.attr("venv_config")(venv); - } - - }catch (const python::error_already_set&) + } + catch (const python::error_already_set&) { PyErr_Print(); } From cb0969c135dfca7eeb3f6c885243096cf1873e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 23 Feb 2024 14:55:57 +0100 Subject: [PATCH 024/156] PythonParser handle missing modules, added type hint --- plugins/python/model/include/model/pyname.h | 1 + plugins/python/parser/pyparser/parser.py | 28 +++++++++++++++------ plugins/python/parser/src/pythonparser.cpp | 1 + 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index d3c247d96..643b894d9 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -27,6 +27,7 @@ struct PYName std::uint64_t column_start; std::uint64_t column_end; std::uint64_t file_id; + std::string type_hint; }; } diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 445455492..8eed454df 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -27,10 +27,10 @@ def parse(path): "nodes": [] } - with open(path) as f: - if config["venv_path"] and path.startswith(config["venv_path"]): - return result + if config["venv_path"] and path.startswith(config["venv_path"]): + return result + with open(path) as f: log(f"Parsing: {path}") source = f.read() script = jedi.Script(source, path=path, project=config["project"]) @@ -44,11 +44,9 @@ def parse(path): if len(defs) > 0: refid = min(list(map(lambda x : hashName(x), defs))) else: - result["status"] = "partial" - log(f"No definition found for {x.full_name}") - log(f"{x.full_name}: file = {x.module_path} line = {x.line} column = {x.column}") refid = hashName(x) - + reportMissingDefinition(x, result) + putInMap(nodes, getNodeInfo(x, refid)) for d in defs: @@ -76,9 +74,20 @@ def getNodeInfo(name, refid): node["is_definition"] = name.is_definition() node["is_builtin"] = name.in_builtin_module() node["file_id"] = getFileId(name) + node["type_hint"] = getNameTypeHint(name) return node +def getNameTypeHint(name): + hint = "" + try: + res = name.get_type_hint() + hint = res if res else "" + except: + pass + + return hint + def getNamePosInfo(name): pos = { "line_start": 0, @@ -97,6 +106,11 @@ def getNamePosInfo(name): return pos +def reportMissingDefinition(name, result): + if name.is_definition() and name.type == 'module' and getNamePosInfo(name)["line_start"] > 0: + log(f"Missing {name.description}") + result["status"] = "partial" + def putInMap(hashmap, node): hashmap[node["id"]] = node diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 3d285fc2c..ad70c22a9 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -80,6 +80,7 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) pyname.file_id = python::extract(node["file_id"]); pyname.value = python::extract(node["value"]); pyname.type = python::extract(node["type"]); + pyname.type_hint = python::extract(node["type_hint"]); // Put in map if(map.find(pyname.id) == map.end()) From aafc8c40ce193e320b0911b6670f67f4129427b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 23 Feb 2024 19:03:05 +0100 Subject: [PATCH 025/156] PythonService display properties. --- plugins/python/parser/pyparser/parser.py | 12 ++++++---- .../service/include/service/pythonservice.h | 1 + plugins/python/service/src/pythonservice.cpp | 22 ++++++++++++++++++- plugins/python/webgui/js/pythonInfoTree.js | 11 +++++++++- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 8eed454df..406cceb43 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -47,7 +47,7 @@ def parse(path): refid = hashName(x) reportMissingDefinition(x, result) - putInMap(nodes, getNodeInfo(x, refid)) + putInMap(nodes, getNodeInfo(x, refid, defs)) for d in defs: putInMap(nodes, getNodeInfo(d, refid)) @@ -59,7 +59,7 @@ def parse(path): return result -def getNodeInfo(name, refid): +def getNodeInfo(name, refid, defs = []): node = {} node["id"] = hashName(name) node["ref_id"] = refid @@ -72,9 +72,9 @@ def getNodeInfo(name, refid): node["type"] = name.type node["is_definition"] = name.is_definition() - node["is_builtin"] = name.in_builtin_module() node["file_id"] = getFileId(name) node["type_hint"] = getNameTypeHint(name) + node["is_builtin"] = name.in_builtin_module() or any(list(map(lambda x : x.in_builtin_module(), defs))) return node @@ -102,7 +102,11 @@ def getNamePosInfo(name): pos["line_end"] = name.get_definition_end_position()[0] pos["column_start"] = name.get_definition_start_position()[1] + 1 pos["column_end"] = name.get_definition_end_position()[1] + 1 - pos["value"] = name.get_line_code() + + if pos["line_start"] == pos["line_end"]: + pos["value"] = name.get_line_code()[pos["column_start"]-1:pos["column_end"]-1] + else: + pos["value"] = name.get_line_code()[pos["column_start"]-1:] return pos diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 59733a9b1..eedb89302 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -255,6 +255,7 @@ class PythonServiceHandler : virtual public LanguageServiceIf std::shared_ptr _datadir; const cc::webserver::ServerContext& _context; + inline const char* const boolToString(bool b) { return b ? "true" : "false"; } }; } // language diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 7828eb94a..7a9df7ceb 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -48,7 +48,7 @@ void PythonServiceHandler::getAstNodeInfoByPosition( model::PYName pyname = *nodes.begin(); return_.id = std::to_string(pyname.id); return_.astNodeValue = pyname.value; - return_.entityHash = 68; + return_.symbolType = pyname.type; }else{ LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos_.pos.line << " column = " << fpos_.pos.column << ")"; } @@ -77,6 +77,26 @@ void PythonServiceHandler::getProperties( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; + _transaction([&]() { + auto nodes = _db->query(odb::query::id == std::stoull(astNodeId_)); + + if(!nodes.empty()) + { + model::PYName pyname = *nodes.begin(); + if(!pyname.full_name.empty()) + { + return_.emplace("Full name", pyname.full_name); + } + + return_.emplace("Builtin", PythonServiceHandler::boolToString(pyname.is_builtin)); + + if(!pyname.type_hint.empty()) + { + return_.emplace("Type hint", pyname.type_hint); + } + } + }); + return; } diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index 122cf99c0..273b2a2fd 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -321,7 +321,16 @@ require([ var props = model.pythonservice.getProperties(elementInfo.id); for (var propName in props) { - var propId = propName.replace(/ /g, '-'); + var propId; + + switch(propName) + { + case "Builtin": propId = "Declaration"; break; + case "Full name": propId = "Name"; break; + case "Type hint": propId = "Type"; break; + default: propId = propName; + } + var label = '' + propName + ': ' + '' + props[propName] + ''; From 8a6e8e11218b711af18014c668f2a350492f2100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 23 Feb 2024 20:14:41 +0100 Subject: [PATCH 026/156] PythonParser Adding imported files to database. --- plugins/python/parser/pyparser/parser.py | 10 +++++++++- plugins/python/parser/src/pythonparser.cpp | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 406cceb43..d2508c8a7 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -2,6 +2,7 @@ from hashlib import sha1 config = { + "root_path": None, "venv_path": None, "project": None } @@ -10,6 +11,9 @@ def log(msg): print(f"[PythonParser] {msg}") def project_config(root_path, venv_path = None): + + config["root_path"] = root_path + try: if venv_path: jedi.create_environment(venv_path) @@ -24,7 +28,8 @@ def project_config(root_path, venv_path = None): def parse(path): result = { "status": "none", - "nodes": [] + "nodes": [], + "imports": [] } if config["venv_path"] and path.startswith(config["venv_path"]): @@ -51,6 +56,9 @@ def parse(path): for d in defs: putInMap(nodes, getNodeInfo(d, refid)) + + if d.module_path and not str(d.module_path).startswith(config["root_path"]): + result["imports"].append(str(d.module_path)) result["nodes"] = list(nodes.values()) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index ad70c22a9..b8a737eef 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -107,6 +107,15 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) _ctx.srcMgr.updateFile(*pyfile); } + // Additional paths (example: builtin definition paths) + // These files need to be added to db + python::object imports = result["imports"]; + for (int i = 0; i < python::len(imports); i++) + { + std::string p = python::extract(imports[i]); + _ctx.srcMgr.getFile(p); + } + }catch (const python::error_already_set&) { PyErr_Print(); From f179caa6653f01a5eda4bca42f8881ce28dc7601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 23 Feb 2024 20:42:55 +0100 Subject: [PATCH 027/156] PythonParser module definition should be the module itself. --- .../service/include/service/pythonservice.h | 1 + plugins/python/service/src/pythonservice.cpp | 60 +++++++++++++------ 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index eedb89302..6812761db 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -256,6 +256,7 @@ class PythonServiceHandler : virtual public LanguageServiceIf std::shared_ptr _datadir; const cc::webserver::ServerContext& _context; inline const char* const boolToString(bool b) { return b ? "true" : "false"; } + model::PYName queryNode(const std::string& id); }; } // language diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 7a9df7ceb..46efbac32 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -78,22 +78,18 @@ void PythonServiceHandler::getProperties( { LOG(info) << "[PYSERVICE] " << __func__; _transaction([&]() { - auto nodes = _db->query(odb::query::id == std::stoull(astNodeId_)); - - if(!nodes.empty()) + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + + if(!pyname.full_name.empty()) { - model::PYName pyname = *nodes.begin(); - if(!pyname.full_name.empty()) - { - return_.emplace("Full name", pyname.full_name); - } + return_.emplace("Full name", pyname.full_name); + } - return_.emplace("Builtin", PythonServiceHandler::boolToString(pyname.is_builtin)); + return_.emplace("Builtin", PythonServiceHandler::boolToString(pyname.is_builtin)); - if(!pyname.type_hint.empty()) - { - return_.emplace("Type hint", pyname.type_hint); - } + if(!pyname.type_hint.empty()) + { + return_.emplace("Type hint", pyname.type_hint); } }); @@ -171,16 +167,21 @@ void PythonServiceHandler::getReferences( _transaction([&]() { - std::uint64_t ref_id = _db->query_value(odb::query::id == std::stoull(astNodeId_)).ref_id; + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); odb::result nodes; switch (referenceId_) { case DEFINITION: - nodes = _db->query(odb::query::ref_id == ref_id && odb::query::is_definition == true); + if (pyname.type != "module") + { + nodes = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true); + }else{ + nodes = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::line_start == 0); + } break; case USAGE: - nodes = _db->query(odb::query::ref_id == ref_id && odb::query::is_definition == false); + nodes = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false); break; } @@ -214,15 +215,20 @@ std::int32_t PythonServiceHandler::getReferenceCount( std::int32_t ret = 0; _transaction([&]() { - std::uint64_t ref_id = _db->query_value(odb::query::id == std::stoull(astNodeId_)).ref_id; + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); switch (referenceId_) { case DEFINITION: - ret = _db->query(odb::query::ref_id == ref_id && odb::query::is_definition == true).size(); + if (pyname.type != "module") + { + ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true).size(); + }else{ + ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::line_start == 0).size(); + } break; case USAGE: - ret = _db->query(odb::query::ref_id == ref_id && odb::query::is_definition == false).size(); + ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false).size(); break; } return; @@ -288,6 +294,22 @@ void PythonServiceHandler::getSyntaxHighlight( return; } +model::PYName PythonServiceHandler::queryNode(const std::string& id) +{ + model::PYName pyname; + + _transaction([&]() { + auto nodes = _db->query(odb::query::id == std::stoull(id)); + + if(!nodes.empty()) + { + pyname = *nodes.begin(); + } + }); + + return pyname; +} + } // language } // service } // cc From a0e7b45f953e3b68fcb402fd33b5a08629090225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 26 Feb 2024 13:44:27 +0100 Subject: [PATCH 028/156] PythonParser Flags for additional syspath. --- plugins/python/parser/pyparser/parser.py | 9 ++++++-- plugins/python/parser/src/pythonparser.cpp | 27 +++++++++++++++------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index d2508c8a7..f4b1ff271 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -10,7 +10,7 @@ def log(msg): print(f"[PythonParser] {msg}") -def project_config(root_path, venv_path = None): +def project_config(root_path, venv_path, sys_path): config["root_path"] = root_path @@ -19,8 +19,13 @@ def project_config(root_path, venv_path = None): jedi.create_environment(venv_path) config["venv_path"] = venv_path log(f"Using virtual environment: {venv_path}") + else: + venv_path = None + + if sys_path: + log(f"Using additional syspath: {sys_path}") - config["project"] = jedi.Project(path = root_path, environment_path = venv_path) + config["project"] = jedi.Project(path = root_path, environment_path = venv_path, added_sys_path = sys_path) except: log(f"Failed to use virtual environment: {venv_path}") diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index b8a737eef..38ab81442 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace cc { @@ -25,21 +26,29 @@ PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) { PyErr_Print(); } - } void PythonParser::prepareInput(const std::string& root_path) { try { - if (_ctx.options.count("venv")) + std::string venv; + python::list sys_path; + + if(_ctx.options.count("syspath")) { - std::string venv = _ctx.options["venv"].as(); - m_py_module.attr("project_config")(root_path, venv); + std::vector vec = _ctx.options["syspath"].as>(); + for(const std::string& s : vec) + { + sys_path.append(s); + } } - else + + if (_ctx.options.count("venvpath")) { - m_py_module.attr("project_config")(root_path); + venv = _ctx.options["venvpath"].as(); } + + m_py_module.attr("project_config")(root_path, venv, sys_path); }catch (const python::error_already_set&) { @@ -176,8 +185,10 @@ extern "C" { boost::program_options::options_description description("Python Plugin"); description.add_options() - ("venv", po::value(), - "Set 'venv' to specify the project's Python virtual environment path."); + ("venvpath", po::value(), + "Set 'venvpath' to specify the project's Python virtual environment path.") + ("syspath", po::value>(), + "Additional sys path for the parser."); return description; } From aac3d0c7edf4ad12a25f9161fe329ef1671dd962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 27 Feb 2024 16:22:45 +0100 Subject: [PATCH 029/156] PythonService Fix astNodeID --- plugins/python/service/src/pythonservice.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 46efbac32..f0195491c 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -188,7 +188,7 @@ void PythonServiceHandler::getReferences( for(const model::PYName& pyname : nodes) { AstNodeInfo info; - info.id = pyname.id; + info.id = std::to_string(pyname.id); info.astNodeType = pyname.type; info.range.file = std::to_string(pyname.file_id); info.range.range.startpos.line = pyname.line_start; From b2e2f0016a917d138e6fdb98ac758e563cd6a029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 4 Mar 2024 16:48:17 +0100 Subject: [PATCH 030/156] PythonParser multiprocess --- .../include/pythonparser/pythonparser.h | 7 +- plugins/python/parser/pyparser/parser.py | 43 +++++++--- plugins/python/parser/src/pythonparser.cpp | 81 +++++++++---------- util/src/dynamiclibrary.cpp | 2 +- 4 files changed, 72 insertions(+), 61 deletions(-) diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index 185657348..06ba100f8 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -31,11 +31,10 @@ class PythonParser : public AbstractParser std::uint32_t full; }; - bool accept(const std::string& path_); - void parseFile(const std::string& path_, PYNameMap& map); python::object m_py_module; - ParseResult m_parse_result; - void prepareInput(const std::string& root_path); + bool accept(const std::string& path_); + void processFile(const python::object& obj, PYNameMap& map, ParseResult& parse_result); + void parseProject(const std::string& root_path); }; } // parser diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index f4b1ff271..e6bc664c5 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -1,17 +1,23 @@ +import os +import sys import jedi -from hashlib import sha1 +import multiprocessing +from itertools import repeat -config = { - "root_path": None, - "venv_path": None, - "project": None -} +from hashlib import sha1 def log(msg): print(f"[PythonParser] {msg}") -def project_config(root_path, venv_path, sys_path): +def parseProject(root_path, venv_path, sys_path): + config = { + "root_path": None, + "venv_path": None, + "project": None + } + + log(f"Parsing project: {root_path}") config["root_path"] = root_path try: @@ -30,16 +36,31 @@ def project_config(root_path, venv_path, sys_path): except: log(f"Failed to use virtual environment: {venv_path}") -def parse(path): + py_files = [] + + for root, dirs, files in os.walk(root_path): + for file in files: + p = os.path.join(root, file) + ext = os.path.splitext(p)[1] + + if ext and ext.lower() == '.py': + py_files.append(p) + + with multiprocessing.Pool(processes=8) as pool: + results = pool.starmap(parse, zip(py_files, repeat(config))) + return list(filter(lambda e : e, results)) + +def parse(path, config): + if config["venv_path"] and path.startswith(config["venv_path"]): + return None + result = { + "path": path, "status": "none", "nodes": [], "imports": [] } - if config["venv_path"] and path.startswith(config["venv_path"]): - return result - with open(path) as f: log(f"Parsing: {path}") source = f.read() diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 38ab81442..993e4494e 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -28,8 +28,13 @@ PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) } } -void PythonParser::prepareInput(const std::string& root_path) +void PythonParser::parseProject(const std::string& root_path) { + PYNameMap map; + ParseResult parse_result; + parse_result.full = 0; + parse_result.partial = 0; + try { std::string venv; python::list sys_path; @@ -48,12 +53,32 @@ void PythonParser::prepareInput(const std::string& root_path) venv = _ctx.options["venvpath"].as(); } - m_py_module.attr("project_config")(root_path, venv, sys_path); + python::object result_list = m_py_module.attr("parseProject")(root_path, venv, sys_path); + for(int i = 0; i < python::len(result_list); i++) + { + PythonParser::processFile(result_list[i], map, parse_result); + } + }catch (const python::error_already_set&) { PyErr_Print(); } + + // Insert into database + for(const auto& [key, value] : map) + { + LOG(debug) << "Inserting PYName " << value.id; + cc::util::OdbTransaction {_ctx.db} ([&] + { + _ctx.db->persist(value); + }); + } + + LOG(info) << "[pythonparser] Parsing finished!"; + LOG(info) << "[pythonparser] Inserted PYName: " << map.size(); + LOG(info) << "[pythonparser] Fully parsed files: " << parse_result.full; + LOG(info) << "[pythonparser] Partially parsed files: " << parse_result.partial; } bool PythonParser::accept(const std::string& path_) @@ -62,14 +87,12 @@ bool PythonParser::accept(const std::string& path_) return ext == ".py"; } -void PythonParser::parseFile(const std::string& path_, PYNameMap& map) +void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseResult& parse_result) { try { - - // Call PythonParser parse(path) - python::object result = m_py_module.attr("parse")(path_); - python::object nodes = result["nodes"]; - std::string status = python::extract(result["status"]); + python::object nodes = obj["nodes"]; + const std::string status = python::extract(obj["status"]); + const std::string path = python::extract(obj["path"]); const int len = python::len(nodes); for (int i = 0; i < len; i++) @@ -100,15 +123,15 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) if(status != "none") { - model::FilePtr pyfile = _ctx.srcMgr.getFile(path_); + model::FilePtr pyfile = _ctx.srcMgr.getFile(path); if(status == "full") { - m_parse_result.full++; + parse_result.full++; pyfile->parseStatus = model::File::ParseStatus::PSFullyParsed; }else if (status == "partial") { - m_parse_result.partial++; + parse_result.partial++; pyfile->parseStatus = model::File::ParseStatus::PSPartiallyParsed; } @@ -118,7 +141,7 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) // Additional paths (example: builtin definition paths) // These files need to be added to db - python::object imports = result["imports"]; + python::object imports = obj["imports"]; for (int i = 0; i < python::len(imports); i++) { std::string p = python::extract(imports[i]); @@ -133,43 +156,11 @@ void PythonParser::parseFile(const std::string& path_, PYNameMap& map) bool PythonParser::parse() { - PYNameMap map; - m_parse_result.full = 0; - m_parse_result.partial = 0; - for(std::string path : _ctx.options["input"].as>()) { - PythonParser::prepareInput(path); - - if(boost::filesystem::is_directory(path)) - { - util::iterateDirectoryRecursive(path, [&](const std::string& currPath_) - { - if (boost::filesystem::is_regular_file(currPath_) && accept(currPath_)) - { - PythonParser::parseFile(currPath_, map); - } - - return true; - }); - } + PythonParser::parseProject(path); } - // Insert into database - for(const auto& [key, value] : map) - { - LOG(debug) << "Inserting PYName " << value.id; - cc::util::OdbTransaction {_ctx.db} ([&] - { - _ctx.db->persist(value); - }); - } - - LOG(info) << "[pythonparser] Parsing finished!"; - LOG(info) << "[pythonparser] Inserted PYName: " << map.size(); - LOG(info) << "[pythonparser] Fully parsed files: " << m_parse_result.full; - LOG(info) << "[pythonparser] Partially parsed files: " << m_parse_result.partial; - return true; } diff --git a/util/src/dynamiclibrary.cpp b/util/src/dynamiclibrary.cpp index 251af2268..4d42c64ef 100644 --- a/util/src/dynamiclibrary.cpp +++ b/util/src/dynamiclibrary.cpp @@ -37,7 +37,7 @@ DynamicLibrary::DynamicLibrary(const std::string& path_) throw std::runtime_error(ss.str()); } #else - _handle = ::dlopen(path_.c_str(), RTLD_NOW); + _handle = ::dlopen(path_.c_str(), RTLD_NOW | RTLD_GLOBAL); if (!_handle) { const char *dlError = ::dlerror(); From 1daabf6ba32d58049392629e23a396c50e69a9e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 5 Mar 2024 13:00:37 +0100 Subject: [PATCH 031/156] PythonParser multiprocess flag --- plugins/python/parser/pyparser/parser.py | 7 ++++--- plugins/python/parser/src/pythonparser.cpp | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index e6bc664c5..142d31ad3 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -9,8 +9,7 @@ def log(msg): print(f"[PythonParser] {msg}") -def parseProject(root_path, venv_path, sys_path): - +def parseProject(root_path, venv_path, sys_path, n_proc): config = { "root_path": None, "venv_path": None, @@ -37,6 +36,8 @@ def parseProject(root_path, venv_path, sys_path): log(f"Failed to use virtual environment: {venv_path}") py_files = [] + + log(f"Using {n_proc} process to parse project") for root, dirs, files in os.walk(root_path): for file in files: @@ -46,7 +47,7 @@ def parseProject(root_path, venv_path, sys_path): if ext and ext.lower() == '.py': py_files.append(p) - with multiprocessing.Pool(processes=8) as pool: + with multiprocessing.Pool(processes=n_proc) as pool: results = pool.starmap(parse, zip(py_files, repeat(config))) return list(filter(lambda e : e, results)) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 993e4494e..4a7ba3c75 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -38,6 +38,7 @@ void PythonParser::parseProject(const std::string& root_path) try { std::string venv; python::list sys_path; + int n_proc = _ctx.options["jobs"].as(); if(_ctx.options.count("syspath")) { @@ -53,7 +54,7 @@ void PythonParser::parseProject(const std::string& root_path) venv = _ctx.options["venvpath"].as(); } - python::object result_list = m_py_module.attr("parseProject")(root_path, venv, sys_path); + python::object result_list = m_py_module.attr("parseProject")(root_path, venv, sys_path, n_proc); for(int i = 0; i < python::len(result_list); i++) { PythonParser::processFile(result_list[i], map, parse_result); @@ -68,7 +69,6 @@ void PythonParser::parseProject(const std::string& root_path) // Insert into database for(const auto& [key, value] : map) { - LOG(debug) << "Inserting PYName " << value.id; cc::util::OdbTransaction {_ctx.db} ([&] { _ctx.db->persist(value); From b082a680189af5cbc6e4af1d47c9857214b807d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 5 Mar 2024 14:52:59 +0100 Subject: [PATCH 032/156] PythonParser logging --- plugins/python/parser/pyparser/parser.py | 10 ++++------ plugins/python/parser/pyparser/parserlog.py | 16 ++++++++++++++++ plugins/python/parser/src/pythonparser.cpp | 3 ++- 3 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 plugins/python/parser/pyparser/parserlog.py diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 142d31ad3..d87e27b31 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -3,12 +3,9 @@ import jedi import multiprocessing from itertools import repeat - +from parserlog import log, bcolors from hashlib import sha1 -def log(msg): - print(f"[PythonParser] {msg}") - def parseProject(root_path, venv_path, sys_path, n_proc): config = { "root_path": None, @@ -146,8 +143,9 @@ def getNamePosInfo(name): return pos def reportMissingDefinition(name, result): - if name.is_definition() and name.type == 'module' and getNamePosInfo(name)["line_start"] > 0: - log(f"Missing {name.description}") + pos = getNamePosInfo(name) + if name.is_definition() and name.type == 'module' and pos["line_start"] > 0: + log(f"{bcolors.FAIL}Missing {name.description} (file = {name.module_path} line = {pos['line_start']})") result["status"] = "partial" def putInMap(hashmap, node): diff --git a/plugins/python/parser/pyparser/parserlog.py b/plugins/python/parser/pyparser/parserlog.py new file mode 100644 index 000000000..54ecdbaa2 --- /dev/null +++ b/plugins/python/parser/pyparser/parserlog.py @@ -0,0 +1,16 @@ +import datetime + +def log(msg): + date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print(f"{date} [pythonparser] {msg}{bcolors.ENDC}") + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' \ No newline at end of file diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 4a7ba3c75..9d7f58e6e 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -67,6 +67,7 @@ void PythonParser::parseProject(const std::string& root_path) } // Insert into database + LOG(info) << "[pythonparser] Inserting PYNames to database..."; for(const auto& [key, value] : map) { cc::util::OdbTransaction {_ctx.db} ([&] @@ -76,7 +77,7 @@ void PythonParser::parseProject(const std::string& root_path) } LOG(info) << "[pythonparser] Parsing finished!"; - LOG(info) << "[pythonparser] Inserted PYName: " << map.size(); + LOG(info) << "[pythonparser] Inserted PYName count: " << map.size(); LOG(info) << "[pythonparser] Fully parsed files: " << parse_result.full; LOG(info) << "[pythonparser] Partially parsed files: " << parse_result.partial; } From cc09bc0952fc15f5cc84626d2fa62e6252ea9df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 5 Mar 2024 16:08:39 +0100 Subject: [PATCH 033/156] PythonParser Fix missing module report --- plugins/python/parser/pyparser/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index d87e27b31..29d2c11ae 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -144,7 +144,7 @@ def getNamePosInfo(name): def reportMissingDefinition(name, result): pos = getNamePosInfo(name) - if name.is_definition() and name.type == 'module' and pos["line_start"] > 0: + if not name.is_definition() and name.type == 'module': log(f"{bcolors.FAIL}Missing {name.description} (file = {name.module_path} line = {pos['line_start']})") result["status"] = "partial" From c13f4d45e771e8794c31ab1b208ffcdf8779f3d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 14 Mar 2024 11:50:46 +0100 Subject: [PATCH 034/156] Include missing headers --- service/workspace/include/workspaceservice/workspaceservice.h | 1 + 1 file changed, 1 insertion(+) diff --git a/service/workspace/include/workspaceservice/workspaceservice.h b/service/workspace/include/workspaceservice/workspaceservice.h index 8926eac2f..9361a67b2 100644 --- a/service/workspace/include/workspaceservice/workspaceservice.h +++ b/service/workspace/include/workspaceservice/workspaceservice.h @@ -2,6 +2,7 @@ #define CC_SERVICE_WORKSPACE_WORKSPACESERVICE_H #include +#include namespace cc { From c5e1b1bfbb1d544a0a9567ad65ec5f9f7ac8696d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 18 Mar 2024 09:41:50 +0100 Subject: [PATCH 035/156] PythonParser simplify parsing finished message --- plugins/python/parser/src/pythonparser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 9d7f58e6e..707e30fcb 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -77,7 +77,7 @@ void PythonParser::parseProject(const std::string& root_path) } LOG(info) << "[pythonparser] Parsing finished!"; - LOG(info) << "[pythonparser] Inserted PYName count: " << map.size(); + LOG(info) << "[pythonparser] Inserted rows: " << map.size(); LOG(info) << "[pythonparser] Fully parsed files: " << parse_result.full; LOG(info) << "[pythonparser] Partially parsed files: " << parse_result.partial; } From 0a87b2b261f4e9737050b6bf73acd53a9a3f8359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 20 Mar 2024 17:32:35 +0100 Subject: [PATCH 036/156] Python syntax highlight --- webgui/scripts/codecompass/view/component/Text.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/webgui/scripts/codecompass/view/component/Text.js b/webgui/scripts/codecompass/view/component/Text.js index 473c820e1..8eacba345 100644 --- a/webgui/scripts/codecompass/view/component/Text.js +++ b/webgui/scripts/codecompass/view/component/Text.js @@ -341,6 +341,11 @@ function (declare, domClass, dom, style, query, topic, ContentPane, Dialog, this.set('content', fileContent); this.set('header', this._fileInfo); + if(this._fileInfo.type == "PY") + { + this._codeMirror.setOption("mode", 'text/x-python'); + } + this._getSyntaxHighlight(this._fileInfo); if (window.gtag) { From 2898a4b19798a604c1f7c96b1c79dd087f9fd791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 20 Mar 2024 18:12:24 +0100 Subject: [PATCH 037/156] PythonService skip imports from definition --- plugins/python/model/include/model/pyname.h | 1 + plugins/python/parser/pyparser/parser.py | 1 + plugins/python/parser/src/pythonparser.cpp | 1 + plugins/python/service/src/pythonservice.cpp | 14 ++------------ 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index 643b894d9..27af37aa8 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -19,6 +19,7 @@ struct PYName std::uint64_t ref_id; bool is_definition = false; bool is_builtin = false; + bool is_import = false; std::string full_name; std::string value; std::string type; diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 29d2c11ae..ca9f9d53b 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -107,6 +107,7 @@ def getNodeInfo(name, refid, defs = []): node["file_id"] = getFileId(name) node["type_hint"] = getNameTypeHint(name) node["is_builtin"] = name.in_builtin_module() or any(list(map(lambda x : x.in_builtin_module(), defs))) + node["is_import"] = "import" in node["value"] return node diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 707e30fcb..b6de43819 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -105,6 +105,7 @@ void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseR pyname.ref_id = python::extract(node["ref_id"]); pyname.full_name = python::extract(node["full_name"]); pyname.is_definition = python::extract(node["is_definition"]); + pyname.is_import = python::extract(node["is_import"]); pyname.is_builtin = python::extract(node["is_builtin"]); pyname.line_start = python::extract(node["line_start"]); pyname.line_end = python::extract(node["line_end"]); diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index f0195491c..c4f44c8d4 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -173,12 +173,7 @@ void PythonServiceHandler::getReferences( switch (referenceId_) { case DEFINITION: - if (pyname.type != "module") - { - nodes = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true); - }else{ - nodes = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::line_start == 0); - } + nodes = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false); break; case USAGE: nodes = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false); @@ -220,12 +215,7 @@ std::int32_t PythonServiceHandler::getReferenceCount( switch (referenceId_) { case DEFINITION: - if (pyname.type != "module") - { - ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true).size(); - }else{ - ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::line_start == 0).size(); - } + ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false).size(); break; case USAGE: ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false).size(); From e71f85c5dd07aaad375e847873d84a84a6ff307e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 25 Mar 2024 10:42:35 +0100 Subject: [PATCH 038/156] PythonService order results by line_start --- plugins/python/service/src/pythonservice.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index c4f44c8d4..987eaaebb 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -173,10 +173,10 @@ void PythonServiceHandler::getReferences( switch (referenceId_) { case DEFINITION: - nodes = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false); + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false) + "ORDER BY" + odb::query::line_start); break; case USAGE: - nodes = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false); + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false) + "ORDER BY" + odb::query::line_start); break; } From b4056b3129201a847b2d6eb82af179cd8c344a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 25 Mar 2024 11:06:34 +0100 Subject: [PATCH 039/156] PythonService Set range for selected node --- plugins/python/service/src/pythonservice.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 987eaaebb..5f8fc3895 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -49,6 +49,11 @@ void PythonServiceHandler::getAstNodeInfoByPosition( return_.id = std::to_string(pyname.id); return_.astNodeValue = pyname.value; return_.symbolType = pyname.type; + return_.range.file = std::to_string(pyname.file_id); + return_.range.range.startpos.line = pyname.line_start; + return_.range.range.startpos.column = pyname.column_start; + return_.range.range.endpos.line = pyname.line_end; + return_.range.range.endpos.column = pyname.column_end; }else{ LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos_.pos.line << " column = " << fpos_.pos.column << ")"; } From 72b868943948cda14088d5771ec418d902c99b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 26 Mar 2024 13:10:27 +0100 Subject: [PATCH 040/156] PythonParser Parent PYName --- plugins/python/model/include/model/pyname.h | 1 + plugins/python/parser/pyparser/parser.py | 3 +++ plugins/python/parser/src/pythonparser.cpp | 1 + 3 files changed, 5 insertions(+) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index 27af37aa8..ef0fc753a 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -17,6 +17,7 @@ struct PYName std::uint64_t id = 0; std::uint64_t ref_id; + std::uint64_t parent; bool is_definition = false; bool is_builtin = false; bool is_import = false; diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index ca9f9d53b..164f90dba 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -109,6 +109,9 @@ def getNodeInfo(name, refid, defs = []): node["is_builtin"] = name.in_builtin_module() or any(list(map(lambda x : x.in_builtin_module(), defs))) node["is_import"] = "import" in node["value"] + parent = name.parent() + node["parent"] = hashName(parent) if parent else node["id"] + return node def getNameTypeHint(name): diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index b6de43819..225632b66 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -103,6 +103,7 @@ void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseR model::PYName pyname; pyname.id = python::extract(node["id"]); pyname.ref_id = python::extract(node["ref_id"]); + pyname.parent = python::extract(node["parent"]); pyname.full_name = python::extract(node["full_name"]); pyname.is_definition = python::extract(node["is_definition"]); pyname.is_import = python::extract(node["is_import"]); From 6589b755ecf1016f5245463e0a54bc2cd6771810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 26 Mar 2024 13:11:14 +0100 Subject: [PATCH 041/156] PythonService Class methods and data members --- plugins/python/service/src/pythonservice.cpp | 29 +++++++++++-- plugins/python/webgui/js/pythonInfoTree.js | 44 -------------------- 2 files changed, 25 insertions(+), 48 deletions(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 5f8fc3895..031bd40aa 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -158,6 +158,14 @@ void PythonServiceHandler::getReferenceTypes( LOG(info) << "[PYSERVICE] " << __func__; return_.emplace("Definition", DEFINITION); return_.emplace("Usage", USAGE); + + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); + if(pyname.type == "class") + { + return_.emplace("Method", METHOD); + return_.emplace("Data member", DATA_MEMBER); + } + return; } @@ -175,13 +183,21 @@ void PythonServiceHandler::getReferences( model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); odb::result nodes; + const odb::query order_by = "ORDER BY" + odb::query::line_start; + switch (referenceId_) { case DEFINITION: - nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false) + "ORDER BY" + odb::query::line_start); + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false) + order_by); break; case USAGE: - nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false) + "ORDER BY" + odb::query::line_start); + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false) + order_by); + break; + case METHOD: + nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "function" && odb::query::is_definition == true) + order_by); + break; + case DATA_MEMBER: + nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "statement" && odb::query::is_definition == true) + order_by); break; } @@ -196,6 +212,7 @@ void PythonServiceHandler::getReferences( info.range.range.endpos.line = pyname.line_end; info.range.range.endpos.column = pyname.column_end; info.astNodeValue = pyname.value; + return_.push_back(info); } @@ -225,6 +242,12 @@ std::int32_t PythonServiceHandler::getReferenceCount( case USAGE: ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false).size(); break; + case METHOD: + ret = _db->query(odb::query::parent == pyname.id && odb::query::type == "function" && odb::query::is_definition == true).size(); + break; + case DATA_MEMBER: + ret = _db->query(odb::query::parent == pyname.id && odb::query::type == "statement" && odb::query::is_definition == true).size(); + break; } return; }); @@ -259,8 +282,6 @@ void PythonServiceHandler::getFileReferenceTypes( const core::FileId& fileId_) { LOG(info) << "[PYSERVICE] " << __func__; - return_["Definition"] = DEFINITION; - return_["Usage"] = USAGE; return; } diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index 273b2a2fd..58f91009f 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -83,46 +83,6 @@ require([ null; } - function groupReferencesByVisibilities(references, parentNode, nodeInfo) { - var res = []; - var visibilities = ['public', 'private', 'protected']; - - visibilities.forEach(function (visibility) { - var nodes = references.filter(function (reference) { - return reference.tags.indexOf(visibility) > -1; - }); - - if (!nodes.length) - return; - - res.push({ - id : nodeInfo.id + visibility + parentNode.refType, - name : createReferenceCountLabel(visibility, nodes.length), - refType : parentNode.refType, - hasChildren : true, - cssClass : 'icon-visibility icon-' + visibility, - getChildren : function () { - var res = []; - - nodes.forEach(function (reference) { - res.push({ - id : visibility + reference.id, - name : createLabel(reference), - refType : parentNode.refType, - nodeInfo : reference, - hasChildren : false, - cssClass : getCssClass(reference) - }); - }); - - return res; - } - }); - }); - - return res; - } - function loadReferenceNodes(parentNode, nodeInfo, refTypes, scratch) { var res = []; var fileGroupsId = []; @@ -133,10 +93,6 @@ require([ nodeInfo.id, parentNode.refType); - if (parentNode.refType === refTypes['Method'] || - parentNode.refType === refTypes['Data member']) - return groupReferencesByVisibilities(references, parentNode, nodeInfo); - references.forEach(function (reference) { if (parentNode.refType === refTypes['Caller'] || parentNode.refType === refTypes['Usage']) { From 2987af77a3e64907ab0421b723fe2255acabaab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 27 Mar 2024 13:23:59 +0100 Subject: [PATCH 042/156] PythonService Added utility functions --- .../service/include/service/pythonservice.h | 2 + plugins/python/service/src/pythonservice.cpp | 121 ++++++++---------- 2 files changed, 52 insertions(+), 71 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 6812761db..022666c90 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -257,6 +257,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf const cc::webserver::ServerContext& _context; inline const char* const boolToString(bool b) { return b ? "true" : "false"; } model::PYName queryNode(const std::string& id); + odb::result queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); + void setInfoProperties(AstNodeInfo& info, const model::PYName& pyname); }; } // language diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 031bd40aa..83d116ac1 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -36,7 +36,7 @@ void PythonServiceHandler::getAstNodeInfoByPosition( { LOG(info) << "[PYSERVICE] " << __func__; _transaction([&]() { - auto nodes = _db->query( + odb::result nodes = _db->query( odb::query::file_id == std::stoull(fpos_.file) && odb::query::line_start == fpos_.pos.line && odb::query::column_start <= fpos_.pos.column && @@ -45,15 +45,7 @@ void PythonServiceHandler::getAstNodeInfoByPosition( if(!nodes.empty()) { - model::PYName pyname = *nodes.begin(); - return_.id = std::to_string(pyname.id); - return_.astNodeValue = pyname.value; - return_.symbolType = pyname.type; - return_.range.file = std::to_string(pyname.file_id); - return_.range.range.startpos.line = pyname.line_start; - return_.range.range.startpos.column = pyname.column_start; - return_.range.range.endpos.line = pyname.line_end; - return_.range.range.endpos.column = pyname.column_end; + PythonServiceHandler::setInfoProperties(return_, *nodes.begin()); }else{ LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos_.pos.line << " column = " << fpos_.pos.column << ")"; } @@ -179,46 +171,16 @@ void PythonServiceHandler::getReferences( LOG(info) << "astNodeID: " << astNodeId_; _transaction([&]() { - - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); - - odb::result nodes; - const odb::query order_by = "ORDER BY" + odb::query::line_start; - - switch (referenceId_) - { - case DEFINITION: - nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false) + order_by); - break; - case USAGE: - nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false) + order_by); - break; - case METHOD: - nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "function" && odb::query::is_definition == true) + order_by); - break; - case DATA_MEMBER: - nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "statement" && odb::query::is_definition == true) + order_by); - break; - } + odb::result nodes = PythonServiceHandler::queryReferences(astNodeId_, referenceId_); for(const model::PYName& pyname : nodes) { AstNodeInfo info; - info.id = std::to_string(pyname.id); - info.astNodeType = pyname.type; - info.range.file = std::to_string(pyname.file_id); - info.range.range.startpos.line = pyname.line_start; - info.range.range.startpos.column = pyname.column_start; - info.range.range.endpos.line = pyname.line_end; - info.range.range.endpos.column = pyname.column_end; - info.astNodeValue = pyname.value; - + PythonServiceHandler::setInfoProperties(info, pyname); return_.push_back(info); } - - return; }); - + return; } @@ -229,30 +191,9 @@ std::int32_t PythonServiceHandler::getReferenceCount( LOG(info) << "[PYSERVICE] " << __func__; LOG(info) << "astNodeID: " << astNodeId_; - std::int32_t ret = 0; - _transaction([&]() { - - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); - - switch (referenceId_) - { - case DEFINITION: - ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false).size(); - break; - case USAGE: - ret = _db->query(odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false).size(); - break; - case METHOD: - ret = _db->query(odb::query::parent == pyname.id && odb::query::type == "function" && odb::query::is_definition == true).size(); - break; - case DATA_MEMBER: - ret = _db->query(odb::query::parent == pyname.id && odb::query::type == "statement" && odb::query::is_definition == true).size(); - break; - } - return; + return _transaction([&]() { + return PythonServiceHandler::queryReferences(astNodeId_, referenceId_).size(); }); - - return ret; } void PythonServiceHandler::getReferencesInFile( @@ -312,18 +253,56 @@ void PythonServiceHandler::getSyntaxHighlight( model::PYName PythonServiceHandler::queryNode(const std::string& id) { - model::PYName pyname; - - _transaction([&]() { - auto nodes = _db->query(odb::query::id == std::stoull(id)); + return _transaction([&]() + { + model::PYName pyname; + odb::result nodes = _db->query(odb::query::id == std::stoull(id)); if(!nodes.empty()) { pyname = *nodes.begin(); } + + return pyname; }); +} + +odb::result PythonServiceHandler::queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId) +{ + odb::result nodes; - return pyname; + const model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); + const odb::query order_by = "ORDER BY" + odb::query::line_start; + + switch (referenceId) + { + case DEFINITION: + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false) + order_by); + break; + case USAGE: + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false) + order_by); + break; + case METHOD: + nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "function" && odb::query::is_definition == true) + order_by); + break; + case DATA_MEMBER: + nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "statement" && odb::query::is_definition == true) + order_by); + break; + } + + return nodes; +} + +void PythonServiceHandler::setInfoProperties(AstNodeInfo& info, const model::PYName& pyname) +{ + info.id = std::to_string(pyname.id); + info.astNodeValue = pyname.value; + info.symbolType = pyname.type; + info.range.file = std::to_string(pyname.file_id); + info.range.range.startpos.line = pyname.line_start; + info.range.range.startpos.column = pyname.column_start; + info.range.range.endpos.line = pyname.line_end; + info.range.range.endpos.column = pyname.column_end; } } // language From 2fc59f9cb0bd3a0da62e2c55cfbfd9d8e19c3dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 27 Mar 2024 15:18:39 +0100 Subject: [PATCH 043/156] PythonService Added support for class references, function local variables --- plugins/python/service/src/pythonservice.cpp | 24 ++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 83d116ac1..a7ea276ca 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -152,12 +152,27 @@ void PythonServiceHandler::getReferenceTypes( return_.emplace("Usage", USAGE); model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); - if(pyname.type == "class") + + if(pyname.type == "function" && pyname.is_definition) { return_.emplace("Method", METHOD); - return_.emplace("Data member", DATA_MEMBER); + return_.emplace("Local variables", LOCAL_VAR); } + _transaction([&]() { + odb::result nodes = PythonServiceHandler::queryReferences(astNodeId, DEFINITION); + + if(!nodes.empty()) + { + model::PYName def = *nodes.begin(); + if(def.type == "class") + { + return_.emplace("Method", METHOD); + return_.emplace("Data member", DATA_MEMBER); + } + } + }); + return; } @@ -283,10 +298,11 @@ odb::result PythonServiceHandler::queryReferences(const core::Ast nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false) + order_by); break; case METHOD: - nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "function" && odb::query::is_definition == true) + order_by); + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "function" && odb::query::is_definition == true) + order_by); break; + case LOCAL_VAR: case DATA_MEMBER: - nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "statement" && odb::query::is_definition == true) + order_by); + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "statement" && odb::query::is_definition == true) + order_by); break; } From ab10b54561202491cc3c7b22eac91db257090d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 22 Apr 2024 10:46:18 +0200 Subject: [PATCH 044/156] PythonService Added reference list type hints, parent node --- .../service/include/service/pythonservice.h | 2 ++ plugins/python/service/src/pythonservice.cpp | 5 +++++ plugins/python/webgui/css/pythonplugin.css | 15 +++++++++++++++ plugins/python/webgui/js/pythonInfoTree.js | 19 ++++++------------- 4 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 plugins/python/webgui/css/pythonplugin.css diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 022666c90..e28cdfe1b 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -193,6 +193,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf EXPANSION, /*!< Macro expansion. */ UNDEFINITION, /*!< Macro undefinition. */ + + PARENT, }; enum FileReferenceType diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index a7ea276ca..c2035e2ec 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -150,6 +150,7 @@ void PythonServiceHandler::getReferenceTypes( LOG(info) << "[PYSERVICE] " << __func__; return_.emplace("Definition", DEFINITION); return_.emplace("Usage", USAGE); + return_.emplace("Parent", PARENT); model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); @@ -304,6 +305,9 @@ odb::result PythonServiceHandler::queryReferences(const core::Ast case DATA_MEMBER: nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "statement" && odb::query::is_definition == true) + order_by); break; + case PARENT: + nodes = _db->query((odb::query::id == pyname.parent) + order_by); + break; } return nodes; @@ -319,6 +323,7 @@ void PythonServiceHandler::setInfoProperties(AstNodeInfo& info, const model::PYN info.range.range.startpos.column = pyname.column_start; info.range.range.endpos.line = pyname.line_end; info.range.range.endpos.column = pyname.column_end; + info.astNodeType = pyname.type_hint; } } // language diff --git a/plugins/python/webgui/css/pythonplugin.css b/plugins/python/webgui/css/pythonplugin.css new file mode 100644 index 000000000..ba924fdef --- /dev/null +++ b/plugins/python/webgui/css/pythonplugin.css @@ -0,0 +1,15 @@ +.icon-Builtin::before{ + content : '\e017' +} + +.icon-Type-hint::before{ + content : '\e040' +} + +.icon-Full-name::before{ + content: '\e001'; +} + +.icon-Parent::before{ + content : '\e033' +} \ No newline at end of file diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index 58f91009f..8afffc724 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -44,8 +44,11 @@ require([ if (astNodeInfo.tags.indexOf('implicit') > -1) labelClass = 'label-implicit'; - var labelValue = astNodeInfo.astNodeValue; - + var labelValue = astNodeInfo.astNodeValue.trim(); + + if (labelValue.slice(-1) == ':') labelValue = labelValue.substr(0, labelValue.length - 1); + if(astNodeInfo.astNodeType) labelValue += ' : ' + astNodeInfo.astNodeType + ""; + // Create dom node for return type of a function and place it at the end of // signature. if (astNodeInfo.symbolType === 'Function') { @@ -277,16 +280,6 @@ require([ var props = model.pythonservice.getProperties(elementInfo.id); for (var propName in props) { - var propId; - - switch(propName) - { - case "Builtin": propId = "Declaration"; break; - case "Full name": propId = "Name"; break; - case "Type hint": propId = "Type"; break; - default: propId = propName; - } - var label = '' + propName + ': ' + '' + props[propName] + ''; @@ -295,7 +288,7 @@ require([ name : label, parent : 'root', nodeInfo : elementInfo, - cssClass : 'icon-' + propId, + cssClass : 'icon-' + propName.replace(/ /g, '-'), hasChildren : false }); } From ca885c2cd762c88af028719326326a0841c918a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 22 Apr 2024 14:48:23 +0200 Subject: [PATCH 045/156] PythonService Added parameter references --- plugins/python/service/src/pythonservice.cpp | 6 +++++- plugins/python/webgui/js/pythonInfoTree.js | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index c2035e2ec..76a1b3f0a 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -151,6 +151,7 @@ void PythonServiceHandler::getReferenceTypes( return_.emplace("Definition", DEFINITION); return_.emplace("Usage", USAGE); return_.emplace("Parent", PARENT); + return_.emplace("Parameters", PARAMETER); model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); @@ -288,7 +289,7 @@ odb::result PythonServiceHandler::queryReferences(const core::Ast odb::result nodes; const model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); - const odb::query order_by = "ORDER BY" + odb::query::line_start; + const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; switch (referenceId) { @@ -308,6 +309,9 @@ odb::result PythonServiceHandler::queryReferences(const core::Ast case PARENT: nodes = _db->query((odb::query::id == pyname.parent) + order_by); break; + case PARAMETER: + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "param" && odb::query::is_definition == true) + order_by); + break; } return nodes; diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index 8afffc724..8b4910c1b 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -47,7 +47,7 @@ require([ var labelValue = astNodeInfo.astNodeValue.trim(); if (labelValue.slice(-1) == ':') labelValue = labelValue.substr(0, labelValue.length - 1); - if(astNodeInfo.astNodeType) labelValue += ' : ' + astNodeInfo.astNodeType + ""; + if(astNodeInfo.astNodeType && !labelValue.includes(':')) labelValue += ' : ' + astNodeInfo.astNodeType + ""; // Create dom node for return type of a function and place it at the end of // signature. From 4fced18d9b7d7a74c74761a64eda4c468a38ac47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 11 Jul 2024 14:01:58 +0200 Subject: [PATCH 046/156] PythonService Added getAstNodeInfo --- plugins/python/service/src/pythonservice.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 76a1b3f0a..fca17000d 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -27,6 +27,11 @@ void PythonServiceHandler::getAstNodeInfo( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; + _transaction([&]() { + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + + PythonServiceHandler::setInfoProperties(return_, pyname); + }); return; } From c93004d94baf8a937bd9ae667b1a398b317d526e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 25 Jul 2024 14:04:31 +0200 Subject: [PATCH 047/156] PyParser filter Python files before multiprocessing --- plugins/python/parser/pyparser/parser.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 164f90dba..4839415cd 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -44,14 +44,14 @@ def parseProject(root_path, venv_path, sys_path, n_proc): if ext and ext.lower() == '.py': py_files.append(p) + if config["venv_path"]: + py_files = filter(lambda e : not(e.startswith(config["venv_path"])), py_files) + with multiprocessing.Pool(processes=n_proc) as pool: results = pool.starmap(parse, zip(py_files, repeat(config))) - return list(filter(lambda e : e, results)) + return results def parse(path, config): - if config["venv_path"] and path.startswith(config["venv_path"]): - return None - result = { "path": path, "status": "none", @@ -172,4 +172,4 @@ def fnvHash(str): return hash & 0xffffffffffffffff def getFileId(name): - return fnvHash(str(name.module_path)) \ No newline at end of file + return fnvHash(str(name.module_path)) From 23eeb89817cbf4a67eadfaeb151a973622ee7f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 20 Aug 2024 14:54:07 +0200 Subject: [PATCH 048/156] PyParser simplify name position info --- plugins/python/parser/pyparser/parser.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 4839415cd..146593463 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -101,6 +101,8 @@ def getNodeInfo(name, refid, defs = []): pos = getNamePosInfo(name) node.update(pos) # merge pos dictionary + node["column_start"] = node["column_start"] + 1 + node["column_end"] = node["column_end"] + 1 node["type"] = name.type node["is_definition"] = name.is_definition() @@ -136,13 +138,13 @@ def getNamePosInfo(name): if name.get_definition_start_position(): pos["line_start"] = name.get_definition_start_position()[0] pos["line_end"] = name.get_definition_end_position()[0] - pos["column_start"] = name.get_definition_start_position()[1] + 1 - pos["column_end"] = name.get_definition_end_position()[1] + 1 + pos["column_start"] = name.get_definition_start_position()[1] + pos["column_end"] = name.get_definition_end_position()[1] if pos["line_start"] == pos["line_end"]: - pos["value"] = name.get_line_code()[pos["column_start"]-1:pos["column_end"]-1] + pos["value"] = name.get_line_code()[pos["column_start"]:pos["column_end"]] else: - pos["value"] = name.get_line_code()[pos["column_start"]-1:] + pos["value"] = name.get_line_code()[pos["column_start"]:] return pos From e69c29a66eae9674626684fd4a2eb979899f54ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 20 Aug 2024 18:09:32 +0200 Subject: [PATCH 049/156] PyParser refactor --- plugins/python/parser/pyparser/parser.py | 96 +---------------- plugins/python/parser/pyparser/parserutil.py | 9 ++ plugins/python/parser/pyparser/pyname.py | 102 +++++++++++++++++++ 3 files changed, 115 insertions(+), 92 deletions(-) create mode 100644 plugins/python/parser/pyparser/parserutil.py create mode 100644 plugins/python/parser/pyparser/pyname.py diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 146593463..ccd703988 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -1,10 +1,9 @@ import os -import sys import jedi import multiprocessing from itertools import repeat -from parserlog import log, bcolors -from hashlib import sha1 +from parserlog import log +from pyname import PYName def parseProject(root_path, venv_path, sys_path, n_proc): config = { @@ -70,16 +69,10 @@ def parse(path, config): for x in script.get_names(references = True, all_scopes = True): defs = x.goto(follow_imports = True, follow_builtin_imports = True) - if len(defs) > 0: - refid = min(list(map(lambda x : hashName(x), defs))) - else: - refid = hashName(x) - reportMissingDefinition(x, result) - - putInMap(nodes, getNodeInfo(x, refid, defs)) + putInMap(nodes, PYName(x).addDefs(defs, result).getNodeInfo()) for d in defs: - putInMap(nodes, getNodeInfo(d, refid)) + putInMap(nodes, PYName(d).getNodeInfo()) if d.module_path and not str(d.module_path).startswith(config["root_path"]): result["imports"].append(str(d.module_path)) @@ -91,87 +84,6 @@ def parse(path, config): return result -def getNodeInfo(name, refid, defs = []): - node = {} - node["id"] = hashName(name) - node["ref_id"] = refid - node["module_name"] = name.module_name - node["module_path"] = name.module_path - node["full_name"] = name.full_name if name.full_name else "" - - pos = getNamePosInfo(name) - node.update(pos) # merge pos dictionary - node["column_start"] = node["column_start"] + 1 - node["column_end"] = node["column_end"] + 1 - - node["type"] = name.type - node["is_definition"] = name.is_definition() - node["file_id"] = getFileId(name) - node["type_hint"] = getNameTypeHint(name) - node["is_builtin"] = name.in_builtin_module() or any(list(map(lambda x : x.in_builtin_module(), defs))) - node["is_import"] = "import" in node["value"] - - parent = name.parent() - node["parent"] = hashName(parent) if parent else node["id"] - - return node - -def getNameTypeHint(name): - hint = "" - try: - res = name.get_type_hint() - hint = res if res else "" - except: - pass - - return hint - -def getNamePosInfo(name): - pos = { - "line_start": 0, - "line_end": 0, - "column_start": 0, - "column_end": 0, - "value": "" - } - - if name.get_definition_start_position(): - pos["line_start"] = name.get_definition_start_position()[0] - pos["line_end"] = name.get_definition_end_position()[0] - pos["column_start"] = name.get_definition_start_position()[1] - pos["column_end"] = name.get_definition_end_position()[1] - - if pos["line_start"] == pos["line_end"]: - pos["value"] = name.get_line_code()[pos["column_start"]:pos["column_end"]] - else: - pos["value"] = name.get_line_code()[pos["column_start"]:] - - return pos - -def reportMissingDefinition(name, result): - pos = getNamePosInfo(name) - if not name.is_definition() and name.type == 'module': - log(f"{bcolors.FAIL}Missing {name.description} (file = {name.module_path} line = {pos['line_start']})") - result["status"] = "partial" - def putInMap(hashmap, node): hashmap[node["id"]] = node -def hashName(name): - pos = getNamePosInfo(name) - s = f"{name.module_path}|{pos['line_start']}|{pos['line_end']}|{pos['column_start']}|{pos['column_end']}".encode("utf-8") - hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff - return hash - -def fnvHash(str): - hash = 14695981039346656037 - - for c in str: - hash ^= ord(c) - hash *= 1099511628211 - - # see: https://stackoverflow.com/questions/20766813/how-to-convert-signed-to-unsigned-integer-in-python - return hash & 0xffffffffffffffff - -def getFileId(name): - return fnvHash(str(name.module_path)) diff --git a/plugins/python/parser/pyparser/parserutil.py b/plugins/python/parser/pyparser/parserutil.py new file mode 100644 index 000000000..f564bef6f --- /dev/null +++ b/plugins/python/parser/pyparser/parserutil.py @@ -0,0 +1,9 @@ +def fnvHash(str): + hash = 14695981039346656037 + + for c in str: + hash ^= ord(c) + hash *= 1099511628211 + + # see: https://stackoverflow.com/questions/20766813/how-to-convert-signed-to-unsigned-integer-in-python + return hash & 0xffffffffffffffff diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py new file mode 100644 index 000000000..3965b04d5 --- /dev/null +++ b/plugins/python/parser/pyparser/pyname.py @@ -0,0 +1,102 @@ +from jedi.api.classes import Name +from typing import List +from hashlib import sha1 +from parserutil import fnvHash +from parserlog import log, bcolors + +class PYName: + id: int + name: Name + refid: int + defs: List[Name] + + def __init__(self, name: Name): + self.name = name + self.hashName = self.__getHashName() + self.refid = self.hashName + self.defs = [] + + def addDefs(self, defs: List[Name], result): + self.defs = defs + + if len(defs) > 0: + self.refid = min(list(map(lambda e : PYName(e).hashName, defs))) + else: + self.__reportMissingDefinition(result) + + return self + + def getNamePosInfo(self): + pos = { + "line_start": 0, + "line_end": 0, + "column_start": 0, + "column_end": 0, + "value": "" + } + + start_pos = self.name.get_definition_start_position() + end_pos = self.name.get_definition_end_position() + + if start_pos and end_pos: + pos["line_start"] = start_pos[0] + pos["line_end"] = end_pos[0] + pos["column_start"] = start_pos[1] + pos["column_end"] = end_pos[1] + + if pos["line_start"] == pos["line_end"]: + pos["value"] = self.name.get_line_code()[pos["column_start"]:pos["column_end"]] + else: + pos["value"] = self.name.get_line_code()[pos["column_start"]:] + + return pos + + def getNodeInfo(self): + node = {} + node["id"] = self.hashName + node["ref_id"] = self.refid + node["module_name"] = self.name.module_name + node["module_path"] = self.name.module_path + node["full_name"] = self.name.full_name if self.name.full_name else "" + + pos = self.getNamePosInfo() + node.update(pos) # merge pos dictionary + node["column_start"] = node["column_start"] + 1 + node["column_end"] = node["column_end"] + 1 + + node["type"] = self.name.type + node["is_definition"] = self.name.is_definition() + node["file_id"] = self.__getFileId() + node["type_hint"] = self.__getNameTypeHint() + node["is_builtin"] = self.name.in_builtin_module() or any(list(map(lambda x : x.in_builtin_module(), self.defs))) + node["is_import"] = "import" in node["value"] + + parent = self.name.parent() + node["parent"] = PYName(parent).hashName if parent else node["id"] + + return node + + def __getHashName(self) -> int: + pos = self.getNamePosInfo() + s = f"{self.name.module_path}|{pos['line_start']}|{pos['line_end']}|{pos['column_start']}|{pos['column_end']}".encode("utf-8") + hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff + return hash + + def __getFileId(self): + return fnvHash(str(self.name.module_path)) + + def __reportMissingDefinition(self, result): + pos = self.getNamePosInfo() + if not self.name.is_definition() and self.name.type == 'module': + log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.name.module_path} line = {pos['line_start']})") + result["status"] = "partial" + + def __getNameTypeHint(self): + hint = "" + try: + res = self.name.get_type_hint() + hint = res if res else "" + except: + pass + + return hint From 6b88323d9cdf94838e2506c31b0dee89578802a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 20 Aug 2024 21:46:58 +0200 Subject: [PATCH 050/156] PyParser PosInfo, ASTHelper --- plugins/python/model/include/model/pyname.h | 3 +- plugins/python/parser/pyparser/asthelper.py | 35 ++++++++++++++ plugins/python/parser/pyparser/parser.py | 5 +- plugins/python/parser/pyparser/posinfo.py | 14 ++++++ plugins/python/parser/pyparser/pyname.py | 50 ++++++++++++-------- plugins/python/parser/src/pythonparser.cpp | 3 +- plugins/python/service/src/pythonservice.cpp | 7 +++ 7 files changed, 94 insertions(+), 23 deletions(-) create mode 100644 plugins/python/parser/pyparser/asthelper.py create mode 100644 plugins/python/parser/pyparser/posinfo.py diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index ef0fc753a..abaa8bf02 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -21,6 +21,7 @@ struct PYName bool is_definition = false; bool is_builtin = false; bool is_import = false; + bool is_call = false; std::string full_name; std::string value; std::string type; @@ -35,4 +36,4 @@ struct PYName } } -#endif \ No newline at end of file +#endif diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py new file mode 100644 index 000000000..604791085 --- /dev/null +++ b/plugins/python/parser/pyparser/asthelper.py @@ -0,0 +1,35 @@ +import ast +from typing import List +from posinfo import PosInfo + +class ASTHelper: + astNodes: List[ast.AST] + source: str + + def __init__(self, source): + self.astNodes = [] + self.source = source + + try: + tree = ast.parse(source) + self.astNodes = list(ast.walk(tree)) + except: + pass + + def getFunctionCalls(self) -> List[ast.AST]: + return list(filter(lambda e : isinstance(e, ast.Call), self.astNodes)) + + def isFunctionCall(self, pos: PosInfo): + for e in self.getFunctionCalls(): + if (not isinstance(e, ast.Call) or not isinstance(e.func, ast.Name)): + continue + + func = e.func + if (func.lineno == pos.line_start and + func.end_lineno == pos.line_end and + func.col_offset == pos.column_start and + func.end_col_offset == pos.column_end): + return True + + return False + diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index ccd703988..bc371d2f5 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -3,6 +3,7 @@ import multiprocessing from itertools import repeat from parserlog import log +from asthelper import ASTHelper from pyname import PYName def parseProject(root_path, venv_path, sys_path, n_proc): @@ -63,13 +64,15 @@ def parse(path, config): source = f.read() script = jedi.Script(source, path=path, project=config["project"]) + asthelper = ASTHelper(source) + nodes = {} result["status"] = "full" for x in script.get_names(references = True, all_scopes = True): defs = x.goto(follow_imports = True, follow_builtin_imports = True) - putInMap(nodes, PYName(x).addDefs(defs, result).getNodeInfo()) + putInMap(nodes, PYName(x).addDefs(defs, result).addASTHelper(asthelper).getNodeInfo()) for d in defs: putInMap(nodes, PYName(d).getNodeInfo()) diff --git a/plugins/python/parser/pyparser/posinfo.py b/plugins/python/parser/pyparser/posinfo.py new file mode 100644 index 000000000..65b1c1635 --- /dev/null +++ b/plugins/python/parser/pyparser/posinfo.py @@ -0,0 +1,14 @@ +class PosInfo: + line_start: int + line_end: int + column_start: int + column_end: int + value: str + + def __init__(self): + self.line_start = 0 + self.line_end = 0 + self.column_start = 0 + self.column_end = 0 + self.value = "" + diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 3965b04d5..97c230ef6 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -3,18 +3,22 @@ from hashlib import sha1 from parserutil import fnvHash from parserlog import log, bcolors +from asthelper import ASTHelper +from posinfo import PosInfo class PYName: id: int name: Name refid: int defs: List[Name] + asthelper: ASTHelper | None def __init__(self, name: Name): self.name = name self.hashName = self.__getHashName() self.refid = self.hashName self.defs = [] + self.asthelper = None def addDefs(self, defs: List[Name], result): self.defs = defs @@ -26,28 +30,26 @@ def addDefs(self, defs: List[Name], result): return self - def getNamePosInfo(self): - pos = { - "line_start": 0, - "line_end": 0, - "column_start": 0, - "column_end": 0, - "value": "" - } + def addASTHelper(self, asthelper: ASTHelper): + self.asthelper = asthelper + return self + + def getNamePosInfo(self) -> PosInfo: + pos = PosInfo() start_pos = self.name.get_definition_start_position() end_pos = self.name.get_definition_end_position() if start_pos and end_pos: - pos["line_start"] = start_pos[0] - pos["line_end"] = end_pos[0] - pos["column_start"] = start_pos[1] - pos["column_end"] = end_pos[1] + pos.line_start = start_pos[0] + pos.line_end = end_pos[0] + pos.column_start = start_pos[1] + pos.column_end = end_pos[1] - if pos["line_start"] == pos["line_end"]: - pos["value"] = self.name.get_line_code()[pos["column_start"]:pos["column_end"]] + if pos.line_start == pos.line_end: + pos.value = self.name.get_line_code()[pos.column_start:pos.column_end] else: - pos["value"] = self.name.get_line_code()[pos["column_start"]:] + pos.value = self.name.get_line_code()[pos.column_start:] return pos @@ -60,9 +62,11 @@ def getNodeInfo(self): node["full_name"] = self.name.full_name if self.name.full_name else "" pos = self.getNamePosInfo() - node.update(pos) # merge pos dictionary - node["column_start"] = node["column_start"] + 1 - node["column_end"] = node["column_end"] + 1 + node["line_start"] = pos.line_start + node["line_end"] = pos.line_end + node["column_start"] = pos.column_start + 1 + node["column_end"] = pos.column_end + 1 + node["value"] = pos.value node["type"] = self.name.type node["is_definition"] = self.name.is_definition() @@ -74,11 +78,17 @@ def getNodeInfo(self): parent = self.name.parent() node["parent"] = PYName(parent).hashName if parent else node["id"] + node["is_call"] = False + + if self.asthelper: + # Add func call + node["is_call"] = self.asthelper.isFunctionCall(pos) + return node def __getHashName(self) -> int: pos = self.getNamePosInfo() - s = f"{self.name.module_path}|{pos['line_start']}|{pos['line_end']}|{pos['column_start']}|{pos['column_end']}".encode("utf-8") + s = f"{self.name.module_path}|{pos.line_start}|{pos.line_end}|{pos.column_start}|{pos.column_end}".encode("utf-8") hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff return hash @@ -88,7 +98,7 @@ def __getFileId(self): def __reportMissingDefinition(self, result): pos = self.getNamePosInfo() if not self.name.is_definition() and self.name.type == 'module': - log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.name.module_path} line = {pos['line_start']})") + log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.name.module_path} line = {pos.line_start})") result["status"] = "partial" def __getNameTypeHint(self): diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 225632b66..60429c14c 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -116,6 +116,7 @@ void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseR pyname.value = python::extract(node["value"]); pyname.type = python::extract(node["type"]); pyname.type_hint = python::extract(node["type_hint"]); + pyname.is_call = python::extract(node["is_call"]); // Put in map if(map.find(pyname.id) == map.end()) @@ -195,4 +196,4 @@ extern "C" #pragma clang diagnostic pop } // parser -} // cc \ No newline at end of file +} // cc diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index fca17000d..07d36c314 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -93,6 +93,9 @@ void PythonServiceHandler::getProperties( { return_.emplace("Type hint", pyname.type_hint); } + + return_.emplace("Function call", PythonServiceHandler::boolToString(pyname.is_call)); + }); return; @@ -157,6 +160,7 @@ void PythonServiceHandler::getReferenceTypes( return_.emplace("Usage", USAGE); return_.emplace("Parent", PARENT); return_.emplace("Parameters", PARAMETER); + return_.emplace("Caller", CALLER); model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); @@ -317,6 +321,9 @@ odb::result PythonServiceHandler::queryReferences(const core::Ast case PARAMETER: nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "param" && odb::query::is_definition == true) + order_by); break; + case CALLER: + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::is_call == true) + order_by); + break; } return nodes; From 33ff8e7a0fea38549aeeb1db04adb7c05c531d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 31 Aug 2024 21:07:16 +0200 Subject: [PATCH 051/156] PyParser isFunctionCall handle attribute case --- plugins/python/parser/pyparser/asthelper.py | 33 ++++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index 604791085..f5d1047b4 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -1,5 +1,5 @@ import ast -from typing import List +from typing import cast, List from posinfo import PosInfo class ASTHelper: @@ -16,20 +16,31 @@ def __init__(self, source): except: pass - def getFunctionCalls(self) -> List[ast.AST]: - return list(filter(lambda e : isinstance(e, ast.Call), self.astNodes)) + def getFunctionCalls(self) -> List[ast.Call]: + return cast(List[ast.Call], list(filter(lambda e : isinstance(e, ast.Call), self.astNodes))) def isFunctionCall(self, pos: PosInfo): for e in self.getFunctionCalls(): - if (not isinstance(e, ast.Call) or not isinstance(e.func, ast.Name)): - continue - func = e.func - if (func.lineno == pos.line_start and - func.end_lineno == pos.line_end and - func.col_offset == pos.column_start and - func.end_col_offset == pos.column_end): - return True + + if (isinstance(func, ast.Name)): + if (func.lineno == pos.line_start and + func.end_lineno == pos.line_end and + func.col_offset == pos.column_start and + func.end_col_offset == pos.column_end): + return True + + elif (isinstance(func, ast.Attribute)): + if func.end_col_offset: + col_start = func.end_col_offset - len(func.attr) + else: + col_start = 0 + + if (func.end_lineno == pos.line_start and + func.end_lineno == pos.line_end and + col_start == pos.column_start and + func.end_col_offset == pos.column_end): + return True return False From a3bcd215b854cee0a037457e970f58f9632fcd07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 31 Aug 2024 21:42:52 +0200 Subject: [PATCH 052/156] PythonService this calls --- plugins/python/service/src/pythonservice.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 07d36c314..c94f70300 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -168,6 +168,7 @@ void PythonServiceHandler::getReferenceTypes( { return_.emplace("Method", METHOD); return_.emplace("Local variables", LOCAL_VAR); + return_.emplace("This calls", THIS_CALLS); } _transaction([&]() { @@ -322,7 +323,10 @@ odb::result PythonServiceHandler::queryReferences(const core::Ast nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "param" && odb::query::is_definition == true) + order_by); break; case CALLER: - nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::is_call == true) + order_by); + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::is_call == true && odb::query::id != pyname.id) + order_by); + break; + case THIS_CALLS: + nodes = _db->query((odb::query::parent == pyname.id && odb::query::is_call == true) + order_by); break; } From 8ebccbf8e25dfb961752dd3f817b5e898fd8549b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 1 Sep 2024 15:57:26 +0000 Subject: [PATCH 053/156] Webgui Function call icon --- plugins/python/webgui/css/pythonplugin.css | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/python/webgui/css/pythonplugin.css b/plugins/python/webgui/css/pythonplugin.css index ba924fdef..c705ad51c 100644 --- a/plugins/python/webgui/css/pythonplugin.css +++ b/plugins/python/webgui/css/pythonplugin.css @@ -12,4 +12,8 @@ .icon-Parent::before{ content : '\e033' -} \ No newline at end of file +} + +.icon-Function-call::before{ + content : '\e028' +} From 2adcb6438048d248d7471233e690de8949dd7dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 1 Sep 2024 16:20:04 +0000 Subject: [PATCH 054/156] PythonService Usage - ignore same PYName --- plugins/python/service/src/pythonservice.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index c94f70300..486fe81d7 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -307,7 +307,7 @@ odb::result PythonServiceHandler::queryReferences(const core::Ast nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false) + order_by); break; case USAGE: - nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false) + order_by); + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::id != pyname.id) + order_by); break; case METHOD: nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "function" && odb::query::is_definition == true) + order_by); From 9b37f240e00cc3af5a9e2bd0eb1c9181c4ec0534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 3 Sep 2024 20:32:42 +0200 Subject: [PATCH 055/156] PythonPlugin venv --- Config.cmake | 3 +++ plugins/python/parser/.gitignore | 1 + plugins/python/parser/CMakeLists.txt | 21 +++++++++++++++++++-- plugins/python/parser/requirements.txt | 2 ++ plugins/python/parser/src/pythonparser.cpp | 12 ++++++++++-- 5 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 plugins/python/parser/.gitignore create mode 100644 plugins/python/parser/requirements.txt diff --git a/Config.cmake b/Config.cmake index 1032ce379..be9f13749 100644 --- a/Config.cmake +++ b/Config.cmake @@ -51,6 +51,9 @@ set(INSTALL_GEN_DIR "${INSTALL_SCRIPTS_DIR}/generated") # Installation directory for Java libraries set(INSTALL_JAVA_LIB_DIR "${INSTALL_LIB_DIR}/java") +# Installation directory for the Python plugin +set(INSTALL_PYTHON_DIR "${INSTALL_LIB_DIR}/pythonplugin") + # Installation directory for executables set(INSTALL_BIN_DIR "bin") diff --git a/plugins/python/parser/.gitignore b/plugins/python/parser/.gitignore new file mode 100644 index 000000000..f7275bbbd --- /dev/null +++ b/plugins/python/parser/.gitignore @@ -0,0 +1 @@ +venv/ diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index b1d6bc7be..e0f7a1c54 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -23,7 +23,24 @@ target_link_libraries(pythonparser target_compile_options(pythonparser PUBLIC -Wno-unknown-pragmas) +set(VENV_DIR "${PLUGIN_DIR}/parser/venv/") +if(NOT EXISTS ${VENV_DIR}) + message("Creating Python virtual environment: ${VENV_DIR}") + execute_process( + COMMAND python3 -m venv venv + WORKING_DIRECTORY ${PLUGIN_DIR}/parser/) +endif() + +message("Installing Python dependencies...") +execute_process( + COMMAND venv/bin/pip install -r requirements.txt + WORKING_DIRECTORY ${PLUGIN_DIR}/parser/) + install(TARGETS pythonparser DESTINATION ${INSTALL_PARSER_DIR}) install( - DIRECTORY ${PLUGIN_DIR}/parser/pyparser/ - DESTINATION ${INSTALL_PARSER_DIR}/pyparser/) + DIRECTORY pyparser/ + DESTINATION ${INSTALL_PYTHON_DIR}/pyparser) +install( + DIRECTORY venv/ + DESTINATION ${INSTALL_PYTHON_DIR}/venv) + diff --git a/plugins/python/parser/requirements.txt b/plugins/python/parser/requirements.txt new file mode 100644 index 000000000..18d880e9d --- /dev/null +++ b/plugins/python/parser/requirements.txt @@ -0,0 +1,2 @@ +jedi==0.18.0 +parso==0.8.4 diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 60429c14c..c51752513 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -13,9 +13,17 @@ namespace parser PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) { // Init Python Interpreter - std::string py_parser_dir = _ctx.compassRoot + "/lib/parserplugin/pyparser/"; + std::string py_parser_dir = _ctx.compassRoot + "/lib/pythonplugin/pyparser"; + std::string py_venv_dir = _ctx.compassRoot + "/lib/pythonplugin/venv"; + std::string path_env = py_venv_dir + "/bin" + ":" + getenv("PATH"); + + // Set Python module path setenv("PYTHONPATH", py_parser_dir.c_str(), 1); - + + // Activate Python venv + setenv("VIRTUAL_ENV", py_venv_dir.c_str(), 1); + setenv("PATH", path_env.c_str(), 1); + Py_Initialize(); // Init PyParser module From e0f85147d000b679697f111d66c5382e5e6c02c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 3 Sep 2024 20:34:31 +0200 Subject: [PATCH 056/156] ignore .cache --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 66ec4b2d1..1d2dce527 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ nbproject/ # Vim *.swp +# clangd cache +.cache/ ## Build folders build/ From cbbd832abded16d46908cec6ab82181b1ad63aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 4 Sep 2024 17:22:51 +0000 Subject: [PATCH 057/156] PyParser Catch jedi errors --- plugins/python/parser/pyparser/parser.py | 47 ++++++++++++++---------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index bc371d2f5..bcd9e60de 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -1,20 +1,24 @@ import os import jedi import multiprocessing +import traceback from itertools import repeat -from parserlog import log +from parserlog import log, bcolors from asthelper import ASTHelper from pyname import PYName def parseProject(root_path, venv_path, sys_path, n_proc): config = { - "root_path": None, + "debug": True, + "root_path": root_path, "venv_path": None, "project": None } log(f"Parsing project: {root_path}") - config["root_path"] = root_path + + if config["debug"]: + log(f"{bcolors.WARNING}Parsing in debug mode!") try: if venv_path: @@ -54,31 +58,36 @@ def parseProject(root_path, venv_path, sys_path, n_proc): def parse(path, config): result = { "path": path, - "status": "none", + "status": "full", "nodes": [], "imports": [] } + nodes = {} + with open(path) as f: - log(f"Parsing: {path}") - source = f.read() - script = jedi.Script(source, path=path, project=config["project"]) + try: + log(f"Parsing: {path}") + source = f.read() + script = jedi.Script(source, path=path, project=config["project"]) - asthelper = ASTHelper(source) + asthelper = ASTHelper(source) - nodes = {} - result["status"] = "full" + for x in script.get_names(references = True, all_scopes = True): + defs = x.goto(follow_imports = True, follow_builtin_imports = True) - for x in script.get_names(references = True, all_scopes = True): - defs = x.goto(follow_imports = True, follow_builtin_imports = True) - - putInMap(nodes, PYName(x).addDefs(defs, result).addASTHelper(asthelper).getNodeInfo()) + putInMap(nodes, PYName(x).addDefs(defs, result).addASTHelper(asthelper).getNodeInfo()) + + for d in defs: + putInMap(nodes, PYName(d).getNodeInfo()) + + if d.module_path and not str(d.module_path).startswith(config["root_path"]): + result["imports"].append(str(d.module_path)) - for d in defs: - putInMap(nodes, PYName(d).getNodeInfo()) - - if d.module_path and not str(d.module_path).startswith(config["root_path"]): - result["imports"].append(str(d.module_path)) + except: + log(f"{bcolors.FAIL}Failed to parse file: {path}") + if config["debug"]: + traceback.print_exc() result["nodes"] = list(nodes.values()) From 178526ffd04ef160bca35ae3fb591f78a9df1936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 5 Sep 2024 12:06:06 +0200 Subject: [PATCH 058/156] PythonService less transaction --- .../service/include/service/pythonservice.h | 3 +- plugins/python/service/src/pythonservice.cpp | 177 +++++++++--------- 2 files changed, 91 insertions(+), 89 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index e28cdfe1b..51c8b5f43 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -259,7 +259,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf const cc::webserver::ServerContext& _context; inline const char* const boolToString(bool b) { return b ? "true" : "false"; } model::PYName queryNode(const std::string& id); - odb::result queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); + model::PYName queryNodeByPosition(const core::FilePosition& fpos); + std::vector queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); void setInfoProperties(AstNodeInfo& info, const model::PYName& pyname); }; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 486fe81d7..cdaf77d78 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -27,11 +27,9 @@ void PythonServiceHandler::getAstNodeInfo( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; - _transaction([&]() { - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); - PythonServiceHandler::setInfoProperties(return_, pyname); - }); + PythonServiceHandler::setInfoProperties(return_, pyname); return; } @@ -40,21 +38,8 @@ void PythonServiceHandler::getAstNodeInfoByPosition( const core::FilePosition& fpos_) { LOG(info) << "[PYSERVICE] " << __func__; - _transaction([&]() { - odb::result nodes = _db->query( - odb::query::file_id == std::stoull(fpos_.file) && - odb::query::line_start == fpos_.pos.line && - odb::query::column_start <= fpos_.pos.column && - odb::query::column_end >= fpos_.pos.column - ); - - if(!nodes.empty()) - { - PythonServiceHandler::setInfoProperties(return_, *nodes.begin()); - }else{ - LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos_.pos.line << " column = " << fpos_.pos.column << ")"; - } - }); + model::PYName node = PythonServiceHandler::queryNodeByPosition(fpos_); + PythonServiceHandler::setInfoProperties(return_, node); return; } @@ -79,24 +64,21 @@ void PythonServiceHandler::getProperties( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; - _transaction([&]() { - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); - - if(!pyname.full_name.empty()) - { - return_.emplace("Full name", pyname.full_name); - } + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); - return_.emplace("Builtin", PythonServiceHandler::boolToString(pyname.is_builtin)); + if(!pyname.full_name.empty()) + { + return_.emplace("Full name", pyname.full_name); + } - if(!pyname.type_hint.empty()) - { - return_.emplace("Type hint", pyname.type_hint); - } + return_.emplace("Builtin", PythonServiceHandler::boolToString(pyname.is_builtin)); - return_.emplace("Function call", PythonServiceHandler::boolToString(pyname.is_call)); + if(!pyname.type_hint.empty()) + { + return_.emplace("Type hint", pyname.type_hint); + } - }); + return_.emplace("Function call", PythonServiceHandler::boolToString(pyname.is_call)); return; } @@ -171,19 +153,17 @@ void PythonServiceHandler::getReferenceTypes( return_.emplace("This calls", THIS_CALLS); } - _transaction([&]() { - odb::result nodes = PythonServiceHandler::queryReferences(astNodeId, DEFINITION); + std::vector nodes = PythonServiceHandler::queryReferences(astNodeId, DEFINITION); - if(!nodes.empty()) + if(!nodes.empty()) + { + model::PYName def = *nodes.begin(); + if(def.type == "class") { - model::PYName def = *nodes.begin(); - if(def.type == "class") - { - return_.emplace("Method", METHOD); - return_.emplace("Data member", DATA_MEMBER); - } + return_.emplace("Method", METHOD); + return_.emplace("Data member", DATA_MEMBER); } - }); + } return; } @@ -197,16 +177,14 @@ void PythonServiceHandler::getReferences( LOG(info) << "[PYSERVICE] " << __func__; LOG(info) << "astNodeID: " << astNodeId_; - _transaction([&]() { - odb::result nodes = PythonServiceHandler::queryReferences(astNodeId_, referenceId_); + std::vector nodes = PythonServiceHandler::queryReferences(astNodeId_, referenceId_); - for(const model::PYName& pyname : nodes) - { - AstNodeInfo info; - PythonServiceHandler::setInfoProperties(info, pyname); - return_.push_back(info); - } - }); + for(const model::PYName& pyname : nodes) + { + AstNodeInfo info; + PythonServiceHandler::setInfoProperties(info, pyname); + return_.push_back(info); + } return; } @@ -218,9 +196,7 @@ std::int32_t PythonServiceHandler::getReferenceCount( LOG(info) << "[PYSERVICE] " << __func__; LOG(info) << "astNodeID: " << astNodeId_; - return _transaction([&]() { - return PythonServiceHandler::queryReferences(astNodeId_, referenceId_).size(); - }); + return PythonServiceHandler::queryReferences(astNodeId_, referenceId_).size(); } void PythonServiceHandler::getReferencesInFile( @@ -294,43 +270,68 @@ model::PYName PythonServiceHandler::queryNode(const std::string& id) }); } -odb::result PythonServiceHandler::queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId) +model::PYName PythonServiceHandler::queryNodeByPosition(const core::FilePosition& fpos) { - odb::result nodes; + return _transaction([&]() + { + model::PYName pyname; + odb::result nodes = _db->query( + odb::query::file_id == std::stoull(fpos.file) && + odb::query::line_start == fpos.pos.line && + odb::query::column_start <= fpos.pos.column && + odb::query::column_end >= fpos.pos.column + ); - const model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); - const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; + if(!nodes.empty()) + { + pyname = *nodes.begin(); + }else{ + LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos.pos.line << " column = " << fpos.pos.column << ")"; + } - switch (referenceId) - { - case DEFINITION: - nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false) + order_by); - break; - case USAGE: - nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::id != pyname.id) + order_by); - break; - case METHOD: - nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "function" && odb::query::is_definition == true) + order_by); - break; - case LOCAL_VAR: - case DATA_MEMBER: - nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "statement" && odb::query::is_definition == true) + order_by); - break; - case PARENT: - nodes = _db->query((odb::query::id == pyname.parent) + order_by); - break; - case PARAMETER: - nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "param" && odb::query::is_definition == true) + order_by); - break; - case CALLER: - nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::is_call == true && odb::query::id != pyname.id) + order_by); - break; - case THIS_CALLS: - nodes = _db->query((odb::query::parent == pyname.id && odb::query::is_call == true) + order_by); - break; - } + return pyname; + }); +} + +std::vector PythonServiceHandler::queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId) +{ + return _transaction([&](){ + odb::result nodes; - return nodes; + const model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); + const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; + + switch (referenceId) + { + case DEFINITION: + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == true && odb::query::is_import == false) + order_by); + break; + case USAGE: + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::id != pyname.id) + order_by); + break; + case METHOD: + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "function" && odb::query::is_definition == true) + order_by); + break; + case LOCAL_VAR: + case DATA_MEMBER: + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "statement" && odb::query::is_definition == true) + order_by); + break; + case PARENT: + nodes = _db->query((odb::query::id == pyname.parent) + order_by); + break; + case PARAMETER: + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "param" && odb::query::is_definition == true) + order_by); + break; + case CALLER: + nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::is_call == true && odb::query::id != pyname.id) + order_by); + break; + case THIS_CALLS: + nodes = _db->query((odb::query::parent == pyname.id && odb::query::is_call == true) + order_by); + break; + } + + return std::vector(nodes.begin(), nodes.end()); + }); } void PythonServiceHandler::setInfoProperties(AstNodeInfo& info, const model::PYName& pyname) From 42ef5f507a1103f290f7e869957ef949bf1564d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 5 Sep 2024 13:12:55 +0200 Subject: [PATCH 059/156] PyParser Highlight venv path error --- plugins/python/parser/pyparser/parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index bcd9e60de..d0494588b 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -34,7 +34,9 @@ def parseProject(root_path, venv_path, sys_path, n_proc): config["project"] = jedi.Project(path = root_path, environment_path = venv_path, added_sys_path = sys_path) except: - log(f"Failed to use virtual environment: {venv_path}") + log(f"{bcolors.FAIL}Failed to use virtual environment: {venv_path}") + if config["debug"]: + traceback.print_exc() py_files = [] From e8354f2420d940b584255a7e98ce682e2953a8e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 5 Sep 2024 13:59:14 +0200 Subject: [PATCH 060/156] PyParser safe environment config --- plugins/python/parser/pyparser/parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index d0494588b..9fa80ee1b 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -10,6 +10,7 @@ def parseProject(root_path, venv_path, sys_path, n_proc): config = { "debug": True, + "safe_env": False, "root_path": root_path, "venv_path": None, "project": None @@ -22,7 +23,7 @@ def parseProject(root_path, venv_path, sys_path, n_proc): try: if venv_path: - jedi.create_environment(venv_path) + jedi.create_environment(venv_path, safe = config["safe_env"]) config["venv_path"] = venv_path log(f"Using virtual environment: {venv_path}") else: From 2ce35389ec10255989d658916d056efbaf797ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 5 Sep 2024 16:12:17 +0200 Subject: [PATCH 061/156] PyParser Better logging of messages --- plugins/python/parser/pyparser/parser.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 9fa80ee1b..2304cba79 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -21,16 +21,19 @@ def parseProject(root_path, venv_path, sys_path, n_proc): if config["debug"]: log(f"{bcolors.WARNING}Parsing in debug mode!") + if not config["safe_env"]: + log(f"{bcolors.WARNING}Creating Python environment in unsafe mode!") + try: if venv_path: jedi.create_environment(venv_path, safe = config["safe_env"]) config["venv_path"] = venv_path - log(f"Using virtual environment: {venv_path}") + log(f"{bcolors.OKGREEN}Using virtual environment: {venv_path}") else: venv_path = None if sys_path: - log(f"Using additional syspath: {sys_path}") + log(f"{bcolors.OKGREEN}Using additional syspath: {sys_path}") config["project"] = jedi.Project(path = root_path, environment_path = venv_path, added_sys_path = sys_path) @@ -41,7 +44,7 @@ def parseProject(root_path, venv_path, sys_path, n_proc): py_files = [] - log(f"Using {n_proc} process to parse project") + log(f"{bcolors.OKGREEN}Using {n_proc} process to parse project") for root, dirs, files in os.walk(root_path): for file in files: From 7ea2de87b8b04e5c375e18c77d563e5d0a8066cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 10 Sep 2024 16:36:45 +0200 Subject: [PATCH 062/156] [Diagrams] Function call diagram WIP --- plugins/python/service/CMakeLists.txt | 1 + plugins/python/service/src/diagram.cpp | 37 +++++++ plugins/python/service/src/diagram.h | 23 +++++ plugins/python/service/src/pythonservice.cpp | 22 ++++ plugins/python/webgui/js/pythonDiagram.js | 100 +++++++++++++++++++ plugins/python/webgui/js/pythonMenu.js | 45 +++++++++ 6 files changed, 228 insertions(+) create mode 100644 plugins/python/service/src/diagram.cpp create mode 100644 plugins/python/service/src/diagram.h create mode 100644 plugins/python/webgui/js/pythonDiagram.js diff --git a/plugins/python/service/CMakeLists.txt b/plugins/python/service/CMakeLists.txt index e145da72d..343873a73 100644 --- a/plugins/python/service/CMakeLists.txt +++ b/plugins/python/service/CMakeLists.txt @@ -13,6 +13,7 @@ include_directories(SYSTEM add_library(pythonservice SHARED src/pythonservice.cpp + src/diagram.cpp src/plugin.cpp) target_compile_options(pythonservice PUBLIC -Wno-unknown-pragmas) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp new file mode 100644 index 000000000..cbdabfc47 --- /dev/null +++ b/plugins/python/service/src/diagram.cpp @@ -0,0 +1,37 @@ +#include "diagram.h" + +namespace +{ + using namespace cc; + + util::Graph::Node addNode(util::Graph& graph_, const model::PYName& pyname) + { + util::Graph::Node node = graph_.getOrCreateNode(std::to_string(pyname.id)); + graph_.setNodeAttribute(node, "label", pyname.value); + + return node; + } +} + +namespace cc +{ +namespace service +{ +namespace language +{ +namespace python +{ + void getFunctionCallDiagram(util::Graph& graph_, const model::PYName& pyname, const std::vector& calls) + { + util::Graph::Node centerNode = addNode(graph_, pyname); + + for (const model::PYName& node : calls) + { + util::Graph::Node callNode = addNode(graph_, node); + util::Graph::Edge edge = graph_.createEdge(centerNode, callNode); + } + } +} // python +} // language +} // service +} // cc diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h new file mode 100644 index 000000000..5af59600c --- /dev/null +++ b/plugins/python/service/src/diagram.h @@ -0,0 +1,23 @@ +#ifndef CC_SERVICE_PYTHON_DIAGRAM_H +#define CC_SERVICE_PYTHON_DIAGRAM_H + +#include +#include +#include +#include + +namespace cc +{ +namespace service +{ +namespace language +{ +namespace python +{ + void getFunctionCallDiagram(util::Graph& graph_, const model::PYName& pyname, const std::vector& calls); +} // python +} // language +} // service +} // cc + +#endif diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index cdaf77d78..623bad21b 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include "diagram.h" namespace cc { @@ -88,6 +90,13 @@ void PythonServiceHandler::getDiagramTypes( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + + if(pyname.is_definition == true && pyname.type == "function") + { + return_.emplace("Function call", FUNCTION_CALL); + } + return; } @@ -97,6 +106,19 @@ void PythonServiceHandler::getDiagram( const std::int32_t diagramId_) { LOG(info) << "[PYSERVICE] " << __func__; + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + util::Graph graph; + + switch (diagramId_) + { + case FUNCTION_CALL: + language::python::getFunctionCallDiagram(graph, pyname, PythonServiceHandler::queryReferences(astNodeId_, THIS_CALLS)); + break; + } + + if (graph.nodeCount() != 0) + return_ = graph.output(util::Graph::SVG); + return; } diff --git a/plugins/python/webgui/js/pythonDiagram.js b/plugins/python/webgui/js/pythonDiagram.js new file mode 100644 index 000000000..a337ce99e --- /dev/null +++ b/plugins/python/webgui/js/pythonDiagram.js @@ -0,0 +1,100 @@ +require([ + 'dojo/topic', + 'dijit/Menu', + 'dijit/MenuItem', + 'dijit/PopupMenuItem', + 'codecompass/model', + 'codecompass/viewHandler'], +function (topic, Menu, MenuItem, PopupMenuItem, model, viewHandler) { + + model.addService('pythonservice', 'PythonService', LanguageServiceClient); + + var astDiagram = { + id : 'python-ast-diagram', + + getDiagram : function (diagramType, nodeId, callback) { + model.pythonservice.getDiagram(nodeId, diagramType, callback); + }, + + getDiagramLegend : function (diagramType) { + return model.pythonservice.getDiagramLegend(diagramType); + }, + + mouseOverInfo : function (diagramType, nodeId) { + var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); + var range = nodeInfo.range.range; + + return { + fileId : nodeInfo.range.file, + selection : [ + range.startpos.line, + range.startpos.column, + range.endpos.line, + range.endpos.column + ] + }; + } + }; + + viewHandler.registerModule(astDiagram, { + type : viewHandler.moduleType.Diagram + }); + + var fileDiagramHandler = { + id : 'python-file-diagram-handler', + + getDiagram : function (diagramType, nodeId, callback) { + model.pythonservice.getFileDiagram(nodeId, diagramType, callback); + }, + + getDiagramLegend : function (diagramType) { + return model.pythonservice.getFileDiagramLegend(diagramType); + }, + + mouseOverInfo : function (diagramType, nodeId) { + return { + fileId : nodeId, + selection : [1,1,1,1] + }; + } + }; + + viewHandler.registerModule(fileDiagramHandler, { + type : viewHandler.moduleType.Diagram + }); + + var fileDiagrams = { + id : 'python-file-diagrams', + render : function (fileInfo) { + var submenu = new Menu(); + + var diagramTypes = model.pythonservice.getFileDiagramTypes(fileInfo.id); + for (diagramType in diagramTypes) + submenu.addChild(new MenuItem({ + label : diagramType, + type : diagramType, + onClick : function () { + var that = this; + + topic.publish('codecompass/openFile', { fileId : fileInfo.id }); + + topic.publish('codecompass/openDiagram', { + handler : 'python-file-diagram-handler', + diagramType : diagramTypes[that.type], + node : fileInfo.id + }); + } + })); + + if (Object.keys(diagramTypes).length !== 0) + return new PopupMenuItem({ + label : 'Python Diagrams', + popup : submenu + }); + } + }; + + viewHandler.registerModule(fileDiagrams, { + type : viewHandler.moduleType.FileManagerContextMenu + }); +}); diff --git a/plugins/python/webgui/js/pythonMenu.js b/plugins/python/webgui/js/pythonMenu.js index d87d6535e..d3487fe3f 100644 --- a/plugins/python/webgui/js/pythonMenu.js +++ b/plugins/python/webgui/js/pythonMenu.js @@ -103,4 +103,49 @@ function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, urlHandler, vi service : model.pythonservice }); + var diagrams = { + id : 'python-text-diagrams', + render : function (nodeInfo, fileInfo) { + if (!nodeInfo || !fileInfo) + return; + + var submenu = new Menu(); + + var diagramTypes = model.pythonservice.getDiagramTypes(nodeInfo.id); + for (diagramType in diagramTypes) + submenu.addChild(new MenuItem({ + label : diagramType, + type : diagramType, + onClick : function () { + var that = this; + + topic.publish('codecompass/openDiagram', { + handler : 'python-ast-diagram', + diagramType : diagramTypes[that.type], + node : nodeInfo.id + }); + } + })); + + submenu.addChild(new MenuItem({ + label : "CodeBites", + onClick : function () { + topic.publish('codecompass/codebites', { + node : nodeInfo + }); + } + })); + + if (Object.keys(diagramTypes).length !== 0) + return new PopupMenuItem({ + label : 'Diagrams', + popup : submenu + }); + } + }; + + viewHandler.registerModule(diagrams, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); }); From bb7cc86b873b8ab45e5231e8ef64308f53fd781e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 14 Sep 2024 21:37:30 +0200 Subject: [PATCH 063/156] [PythonService] Function call diagram --- .../service/include/service/pythonservice.h | 18 +-- plugins/python/service/src/diagram.cpp | 115 +++++++++++++++--- plugins/python/service/src/diagram.h | 23 +++- plugins/python/service/src/pythonservice.cpp | 38 ++++-- 4 files changed, 159 insertions(+), 35 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 51c8b5f43..bf2b8f9d7 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -3,15 +3,17 @@ #include #include +#include #include -#include #include #include +#include #include +#include #include #include @@ -123,7 +125,6 @@ class PythonServiceHandler : virtual public LanguageServiceIf std::vector& return_, const core::FileRange& range_) override; -private: enum ReferenceType { DEFINITION, /*!< By this option the definition(s) of the AST node can be @@ -252,15 +253,18 @@ class PythonServiceHandler : virtual public LanguageServiceIf of a module. */ }; + model::PYName queryNode(const std::string& id); + model::PYName queryNodeByPosition(const core::FilePosition& fpos); + std::vector queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); + std::string getNodeLineValue(const model::PYName& pyname); + +private: std::shared_ptr _db; util::OdbTransaction _transaction; - std::shared_ptr _datadir; const cc::webserver::ServerContext& _context; - inline const char* const boolToString(bool b) { return b ? "true" : "false"; } - model::PYName queryNode(const std::string& id); - model::PYName queryNodeByPosition(const core::FilePosition& fpos); - std::vector queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); + + inline const char* boolToString(bool b) { return b ? "true" : "false"; } void setInfoProperties(AstNodeInfo& info, const model::PYName& pyname); }; diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index cbdabfc47..9941eee81 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -1,18 +1,5 @@ #include "diagram.h" -namespace -{ - using namespace cc; - - util::Graph::Node addNode(util::Graph& graph_, const model::PYName& pyname) - { - util::Graph::Node node = graph_.getOrCreateNode(std::to_string(pyname.id)); - graph_.setNodeAttribute(node, "label", pyname.value); - - return node; - } -} - namespace cc { namespace service @@ -21,16 +8,108 @@ namespace language { namespace python { - void getFunctionCallDiagram(util::Graph& graph_, const model::PYName& pyname, const std::vector& calls) +Diagram::Diagram( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_) + : m_pythonService(db_, datadir_, context_){} + +util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) +{ + const std::vector callers = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::CALLER); + const std::vector calls = functionGoto(m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::THIS_CALLS)); + + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + util::Graph::Node centerNode = addNode(graph, pyname); + decorateNode(graph, centerNode, FunctionCenterNode); + + for (const model::PYName& node : calls) + { + util::Graph::Node callNode = addNode(graph, node); + decorateNode(graph, callNode, node.is_definition ? FunctionCallDefinitionNode : FunctionCallNode); + util::Graph::Edge edge = graph.createEdge(centerNode, callNode); + } + + for (const model::PYName& node : callers) { - util::Graph::Node centerNode = addNode(graph_, pyname); + util::Graph::Node callerNode = addNode(graph, node); + decorateNode(graph, callerNode, FunctionCallerNode); + util::Graph::Edge edge = graph.createEdge(callerNode, centerNode); + } + + return graph; +} + +std::vector Diagram::functionGoto(const std::vector& functions) +{ + std::vector calls; + std::unordered_map added; - for (const model::PYName& node : calls) + // Find definition for each function + for(const model::PYName& p : functions) + { + std::vector defs = m_pythonService.queryReferences(std::to_string(p.id), PythonServiceHandler::DEFINITION); + if(defs.size() == 1) { - util::Graph::Node callNode = addNode(graph_, node); - util::Graph::Edge edge = graph_.createEdge(centerNode, callNode); + model::PYName d = defs[0]; + + if(d.value.empty()) + { + calls.push_back(p); + continue; + } + + if(added.count(d.id) == 0) + { + calls.push_back(d); + added.emplace(d.id, true); + } + }else{ + calls.push_back(p); } } + + return calls; +} + +util::Graph::Node Diagram::addNode(util::Graph& graph_, const model::PYName& pyname) +{ + util::Graph::Node node = graph_.getOrCreateNode(std::to_string(pyname.id)); + + std::string label = pyname.value; + + if(!pyname.is_definition) + { + label = m_pythonService.getNodeLineValue(pyname); + } + + graph_.setNodeAttribute(node, "label", label); + + return node; +} + +void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeDecoration& decoration) +{ + graph_.setNodeAttribute(node_, "style", "filled"); + + switch(decoration) + { + case FunctionCenterNode: + graph_.setNodeAttribute(node_, "fillcolor", "gold"); + break; + case FunctionCallerNode: + graph_.setNodeAttribute(node_, "fillcolor", "coral"); + break; + case FunctionCallNode: + graph_.setNodeAttribute(node_, "fillcolor", "lightblue"); + break; + case FunctionCallDefinitionNode: + graph_.setNodeAttribute(node_, "fillcolor", "cyan"); + break; + } +} } // python } // language } // service diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 5af59600c..6ef562f49 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace cc { @@ -14,7 +15,27 @@ namespace language { namespace python { - void getFunctionCallDiagram(util::Graph& graph_, const model::PYName& pyname, const std::vector& calls); + class Diagram { + public: + Diagram( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_); + + util::Graph getFunctionCallDiagram(const model::PYName& pyname); + + enum NodeDecoration { + FunctionCenterNode, + FunctionCallNode, + FunctionCallDefinitionNode, + FunctionCallerNode + }; + private: + PythonServiceHandler m_pythonService; + std::vector functionGoto(const std::vector& functions); + void decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeDecoration& decoration); + util::Graph::Node addNode(util::Graph& graph_, const model::PYName& pyname); + }; } // python } // language } // service diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 623bad21b..25079a04a 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -12,9 +12,9 @@ namespace language PythonServiceHandler::PythonServiceHandler( std::shared_ptr db_, - std::shared_ptr /*datadir_*/, + std::shared_ptr datadir_, const cc::webserver::ServerContext& context_) - : _db(db_), _transaction(db_), _context(context_) {} + : _db(db_), _transaction(db_), _datadir(datadir_), _context(context_) {} void PythonServiceHandler::getFileTypes( std::vector& return_) @@ -106,15 +106,19 @@ void PythonServiceHandler::getDiagram( const std::int32_t diagramId_) { LOG(info) << "[PYSERVICE] " << __func__; - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); - util::Graph graph; + python::Diagram diagram(_db, _datadir, _context); - switch (diagramId_) + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + util::Graph graph = [&]() { - case FUNCTION_CALL: - language::python::getFunctionCallDiagram(graph, pyname, PythonServiceHandler::queryReferences(astNodeId_, THIS_CALLS)); - break; - } + switch (diagramId_) + { + case FUNCTION_CALL: + return diagram.getFunctionCallDiagram(pyname); + default: + return util::Graph(); + } + }(); if (graph.nodeCount() != 0) return_ = graph.output(util::Graph::SVG); @@ -369,6 +373,22 @@ void PythonServiceHandler::setInfoProperties(AstNodeInfo& info, const model::PYN info.astNodeType = pyname.type_hint; } +std::string PythonServiceHandler::getNodeLineValue(const model::PYName& pyname) +{ + core::ProjectServiceHandler projectService(_db, _datadir, _context); + std::string content; + projectService.getFileContent(content, std::to_string(pyname.file_id)); + + std::istringstream iss(content); + std::string lineStr; + + for (std::size_t i = 1; i <= pyname.line_start; ++i) + { + std::getline(iss, lineStr); + } + + return lineStr; +} } // language } // service } // cc From 767c621a36a9d996f2af6b83188cd8d5a354c4dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 15 Sep 2024 22:18:46 +0200 Subject: [PATCH 064/156] [Diagrams] Find parent function --- plugins/python/model/include/model/pyname.h | 5 ++++- plugins/python/parser/pyparser/pyname.py | 18 +++++++++++++++ plugins/python/parser/src/pythonparser.cpp | 1 + .../service/include/service/pythonservice.h | 2 ++ plugins/python/service/src/diagram.cpp | 22 ++++++++++++------- plugins/python/service/src/diagram.h | 5 +++-- plugins/python/service/src/pythonservice.cpp | 3 +++ 7 files changed, 45 insertions(+), 11 deletions(-) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index abaa8bf02..ca5c2b8a6 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -18,6 +18,8 @@ struct PYName std::uint64_t ref_id; std::uint64_t parent; + std::uint64_t parent_function; + bool is_definition = false; bool is_builtin = false; bool is_import = false; @@ -25,12 +27,13 @@ struct PYName std::string full_name; std::string value; std::string type; + std::string type_hint; + std::uint64_t line_start; std::uint64_t line_end; std::uint64_t column_start; std::uint64_t column_end; std::uint64_t file_id; - std::string type_hint; }; } diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 97c230ef6..2796065f6 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -84,6 +84,8 @@ def getNodeInfo(self): # Add func call node["is_call"] = self.asthelper.isFunctionCall(pos) + node["parent_function"] = self.__getParentFunction() + return node def __getHashName(self) -> int: @@ -101,6 +103,22 @@ def __reportMissingDefinition(self, result): log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.name.module_path} line = {pos.line_start})") result["status"] = "partial" + def __getParentFunction(self): + try: + node = self.name + for _ in range(0,10): + parent: Name = node.parent() + if parent and parent.type == "function" and parent.is_definition(): + return PYName(parent).hashName + elif parent: + node = parent + else: + break + except: + pass + + return self.hashName + def __getNameTypeHint(self): hint = "" try: diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index c51752513..5c8e78525 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -112,6 +112,7 @@ void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseR pyname.id = python::extract(node["id"]); pyname.ref_id = python::extract(node["ref_id"]); pyname.parent = python::extract(node["parent"]); + pyname.parent_function = python::extract(node["parent_function"]); pyname.full_name = python::extract(node["full_name"]); pyname.is_definition = python::extract(node["is_definition"]); pyname.is_import = python::extract(node["is_import"]); diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index bf2b8f9d7..80aee7774 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -196,6 +196,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf UNDEFINITION, /*!< Macro undefinition. */ PARENT, + + PARENT_FUNCTION, }; enum FileReferenceType diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 9941eee81..1a5073a45 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -16,8 +16,8 @@ Diagram::Diagram( util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) { - const std::vector callers = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::CALLER); - const std::vector calls = functionGoto(m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::THIS_CALLS)); + const std::vector callers = functionGoto(m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::CALLER), PythonServiceHandler::PARENT_FUNCTION); + const std::vector calls = functionGoto(m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::THIS_CALLS), PythonServiceHandler::DEFINITION); util::Graph graph; graph.setAttribute("rankdir", "LR"); @@ -35,22 +35,23 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) for (const model::PYName& node : callers) { util::Graph::Node callerNode = addNode(graph, node); - decorateNode(graph, callerNode, FunctionCallerNode); + decorateNode(graph, callerNode, node.is_definition ? FunctionCallerDefinitionNode : FunctionCallerNode); util::Graph::Edge edge = graph.createEdge(callerNode, centerNode); } return graph; } -std::vector Diagram::functionGoto(const std::vector& functions) +std::vector Diagram::functionGoto(const std::vector& functions, const PythonServiceHandler::ReferenceType& ref_type) { std::vector calls; std::unordered_map added; - // Find definition for each function + // Find reference for each function + // Example use case: Find definition for each function for(const model::PYName& p : functions) { - std::vector defs = m_pythonService.queryReferences(std::to_string(p.id), PythonServiceHandler::DEFINITION); + std::vector defs = m_pythonService.queryReferences(std::to_string(p.id), ref_type); if(defs.size() == 1) { model::PYName d = defs[0]; @@ -100,13 +101,18 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const graph_.setNodeAttribute(node_, "fillcolor", "gold"); break; case FunctionCallerNode: + graph_.setNodeAttribute(node_, "fillcolor", "orange"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; + case FunctionCallerDefinitionNode: graph_.setNodeAttribute(node_, "fillcolor", "coral"); break; case FunctionCallNode: - graph_.setNodeAttribute(node_, "fillcolor", "lightblue"); + graph_.setNodeAttribute(node_, "fillcolor", "cyan"); + graph_.setNodeAttribute(node_, "shape", "box"); break; case FunctionCallDefinitionNode: - graph_.setNodeAttribute(node_, "fillcolor", "cyan"); + graph_.setNodeAttribute(node_, "fillcolor", "lightblue"); break; } } diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 6ef562f49..c65fd49f5 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -28,11 +28,12 @@ namespace python FunctionCenterNode, FunctionCallNode, FunctionCallDefinitionNode, - FunctionCallerNode + FunctionCallerNode, + FunctionCallerDefinitionNode }; private: PythonServiceHandler m_pythonService; - std::vector functionGoto(const std::vector& functions); + std::vector functionGoto(const std::vector& functions, const PythonServiceHandler::ReferenceType& ref_type); void decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeDecoration& decoration); util::Graph::Node addNode(util::Graph& graph_, const model::PYName& pyname); }; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 25079a04a..42ebcc171 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -345,6 +345,9 @@ std::vector PythonServiceHandler::queryReferences(const core::Ast case PARENT: nodes = _db->query((odb::query::id == pyname.parent) + order_by); break; + case PARENT_FUNCTION: + nodes = _db->query((odb::query::id == pyname.parent_function) + order_by); + break; case PARAMETER: nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "param" && odb::query::is_definition == true) + order_by); break; From a6ea04b471cc763d42b69ce9fd09dca1912ca2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 15 Sep 2024 22:31:04 +0200 Subject: [PATCH 065/156] [PythonService] getSourceText --- .../service/include/service/pythonservice.h | 1 + plugins/python/service/src/pythonservice.cpp | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 80aee7774..d4038b1bf 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -8,6 +8,7 @@ #include #include +#include #include #include diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 42ebcc171..62521372c 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -50,6 +50,21 @@ void PythonServiceHandler::getSourceText( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; + model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + + core::ProjectServiceHandler projectService(_db, _datadir, _context); + std::string content; + projectService.getFileContent(content, std::to_string(pyname.file_id)); + + if(!content.empty()) + { + return_ = cc::util::textRange( + content, + pyname.line_start, + pyname.column_start, + pyname.line_end, + pyname.column_end); + } return; } From 0a4a58e69b64986af94b691ca57237d1055f1fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 16 Sep 2024 22:03:35 +0200 Subject: [PATCH 066/156] [PyParser] Add all definition import paths --- plugins/python/parser/pyparser/parser.py | 2 +- plugins/python/parser/src/pythonparser.cpp | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 2304cba79..6e7872a74 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -87,7 +87,7 @@ def parse(path, config): for d in defs: putInMap(nodes, PYName(d).getNodeInfo()) - if d.module_path and not str(d.module_path).startswith(config["root_path"]): + if d.module_path: result["imports"].append(str(d.module_path)) except: diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 5c8e78525..5d244be4e 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -158,7 +158,10 @@ void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseR for (int i = 0; i < python::len(imports); i++) { std::string p = python::extract(imports[i]); - _ctx.srcMgr.getFile(p); + + model::FilePtr file = _ctx.srcMgr.getFile(p); + file->type = "PY"; + _ctx.srcMgr.updateFile(*file); } }catch (const python::error_already_set&) From e243eea58f3056feebe5efe38e345f0e1a3a4cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 17 Sep 2024 12:31:41 +0200 Subject: [PATCH 067/156] [Diagrams] Make certain nodes unclickable --- plugins/python/service/src/diagram.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 1a5073a45..e08697dd6 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -23,12 +23,19 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) graph.setAttribute("rankdir", "LR"); util::Graph::Node centerNode = addNode(graph, pyname); + const std::string centerNodeID = std::to_string(pyname.id); decorateNode(graph, centerNode, FunctionCenterNode); for (const model::PYName& node : calls) { util::Graph::Node callNode = addNode(graph, node); decorateNode(graph, callNode, node.is_definition ? FunctionCallDefinitionNode : FunctionCallNode); + + if(!node.is_definition) + { + graph.setNodeAttribute(callNode, "id", centerNodeID); + } + util::Graph::Edge edge = graph.createEdge(centerNode, callNode); } @@ -36,6 +43,12 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) { util::Graph::Node callerNode = addNode(graph, node); decorateNode(graph, callerNode, node.is_definition ? FunctionCallerDefinitionNode : FunctionCallerNode); + + if(!node.is_definition) + { + graph.setNodeAttribute(callerNode, "id", centerNodeID); + } + util::Graph::Edge edge = graph.createEdge(callerNode, centerNode); } From 50e6f5f15a94c0596b20f94471548e83cd0dc4d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 17 Sep 2024 13:15:20 +0200 Subject: [PATCH 068/156] [Diagrams] Function call diagram file paths --- plugins/python/service/src/diagram.cpp | 31 ++++++++++++++++++++++++-- plugins/python/service/src/diagram.h | 2 ++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index e08697dd6..e1ab7a7c6 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -12,7 +12,8 @@ Diagram::Diagram( std::shared_ptr db_, std::shared_ptr datadir_, const cc::webserver::ServerContext& context_) - : m_pythonService(db_, datadir_, context_){} + : m_pythonService(db_, datadir_, context_), + m_projectService(db_, datadir_, context_){} util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) { @@ -90,7 +91,33 @@ std::vector Diagram::functionGoto(const std::vectorsecond; + + core::FileInfo fileInfo; + m_projectService.getFileInfo(fileInfo, fileId); + + util::Graph::Subgraph subgraph + = graph_.getOrCreateSubgraph("cluster_" + fileInfo.path); + + graph_.setSubgraphAttribute(subgraph, "id", fileInfo.id); + const std::string coloredLabel = + "
" + + fileInfo.path + + "
"; + graph_.setSubgraphAttribute(subgraph, "label", coloredLabel, true); + + m_subgraphs.insert(it, std::make_pair(fileInfo.path, subgraph)); + + return subgraph; + }(); + + util::Graph::Node node = graph_.getOrCreateNode(std::to_string(pyname.id), subgraph); std::string label = pyname.value; diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index c65fd49f5..09ee5c645 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -33,6 +33,8 @@ namespace python }; private: PythonServiceHandler m_pythonService; + core::ProjectServiceHandler m_projectService; + std::map m_subgraphs; std::vector functionGoto(const std::vector& functions, const PythonServiceHandler::ReferenceType& ref_type); void decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeDecoration& decoration); util::Graph::Node addNode(util::Graph& graph_, const model::PYName& pyname); From 61dd32b19df19fea2075415eadc7a6623ef2f09d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 17 Sep 2024 16:41:31 +0200 Subject: [PATCH 069/156] [Diagrams] Fix node decoration types --- plugins/python/service/src/diagram.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index e1ab7a7c6..75ad14ffa 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -30,10 +30,13 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) for (const model::PYName& node : calls) { util::Graph::Node callNode = addNode(graph, node); - decorateNode(graph, callNode, node.is_definition ? FunctionCallDefinitionNode : FunctionCallNode); - - if(!node.is_definition) + if(node.is_definition && node.type == "function") + { + decorateNode(graph, callNode, FunctionCallDefinitionNode); + } + else { + decorateNode(graph, callNode, FunctionCallNode); graph.setNodeAttribute(callNode, "id", centerNodeID); } @@ -43,11 +46,13 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) for (const model::PYName& node : callers) { util::Graph::Node callerNode = addNode(graph, node); - decorateNode(graph, callerNode, node.is_definition ? FunctionCallerDefinitionNode : FunctionCallerNode); - - if(!node.is_definition) + if(node.is_definition && node.type == "function") + { + decorateNode(graph, callerNode, FunctionCallerDefinitionNode); + } + else { - graph.setNodeAttribute(callerNode, "id", centerNodeID); + decorateNode(graph, callerNode, FunctionCallerNode); } util::Graph::Edge edge = graph.createEdge(callerNode, centerNode); From 38326e3997294293058fb19d91b921e2084cd6be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 17 Sep 2024 19:09:04 +0200 Subject: [PATCH 070/156] [PyParser] Use asthelper for is_import --- plugins/python/parser/pyparser/asthelper.py | 15 ++++++++++++++- plugins/python/parser/pyparser/pyname.py | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index f5d1047b4..f2b409a55 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -19,6 +19,9 @@ def __init__(self, source): def getFunctionCalls(self) -> List[ast.Call]: return cast(List[ast.Call], list(filter(lambda e : isinstance(e, ast.Call), self.astNodes))) + def getImports(self) -> List[ast.Import | ast.ImportFrom]: + return cast(List[ast.Import | ast.ImportFrom], list(filter(lambda e : isinstance(e, ast.Import) or isinstance(e, ast.ImportFrom), self.astNodes))) + def isFunctionCall(self, pos: PosInfo): for e in self.getFunctionCalls(): func = e.func @@ -43,4 +46,14 @@ def isFunctionCall(self, pos: PosInfo): return True return False - + + def isImport(self, pos: PosInfo): + for e in self.getImports(): + if (e.lineno == pos.line_start and + e.end_lineno == pos.line_end and + e.col_offset == pos.column_start and + e.end_col_offset == pos.column_end): + return True + + return False + diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 2796065f6..d34219cab 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -73,16 +73,16 @@ def getNodeInfo(self): node["file_id"] = self.__getFileId() node["type_hint"] = self.__getNameTypeHint() node["is_builtin"] = self.name.in_builtin_module() or any(list(map(lambda x : x.in_builtin_module(), self.defs))) - node["is_import"] = "import" in node["value"] parent = self.name.parent() node["parent"] = PYName(parent).hashName if parent else node["id"] node["is_call"] = False + node["is_import"] = False if self.asthelper: - # Add func call node["is_call"] = self.asthelper.isFunctionCall(pos) + node["is_import"] = self.asthelper.isImport(pos) node["parent_function"] = self.__getParentFunction() From 3b4bd8ddf78d15aa614328a66729ad0f17eb5535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 17 Sep 2024 19:30:51 +0200 Subject: [PATCH 071/156] [Diagrams] Fix node decoration types again --- plugins/python/service/src/diagram.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 75ad14ffa..25f2aa01b 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -30,7 +30,7 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) for (const model::PYName& node : calls) { util::Graph::Node callNode = addNode(graph, node); - if(node.is_definition && node.type == "function") + if(node.is_definition && (node.type == "function" || node.type == "class")) { decorateNode(graph, callNode, FunctionCallDefinitionNode); } @@ -46,7 +46,7 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) for (const model::PYName& node : callers) { util::Graph::Node callerNode = addNode(graph, node); - if(node.is_definition && node.type == "function") + if(node.is_definition && (node.type == "function" || node.type == "class")) { decorateNode(graph, callerNode, FunctionCallerDefinitionNode); } From 90805421aa98dc3bcd5f46deb47530aa111c1a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 18 Sep 2024 00:32:17 +0200 Subject: [PATCH 072/156] [Model] Database indexes --- plugins/python/model/include/model/pyname.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index ca5c2b8a6..1ef3e43de 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -13,10 +13,12 @@ namespace model #pragma db object struct PYName { - #pragma db id + #pragma db id unique std::uint64_t id = 0; + #pragma db index std::uint64_t ref_id; + std::uint64_t parent; std::uint64_t parent_function; @@ -33,6 +35,8 @@ struct PYName std::uint64_t line_end; std::uint64_t column_start; std::uint64_t column_end; + + #pragma db index std::uint64_t file_id; }; From 881e3d0033b5c99e3af993a9d1fb8e23a5948022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 18 Sep 2024 11:32:47 +0200 Subject: [PATCH 073/156] [PythonService] queryNodes, module diagram --- .../service/include/service/pythonservice.h | 6 ++- plugins/python/service/src/pythonservice.cpp | 54 ++++++++++++++++--- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index d4038b1bf..80e08b788 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -219,6 +219,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf the edges are the function calls between them. The diagram also displays some dynamic information such as virtual function calls. */ + MODULE_DEPENDENCY, + DETAILED_CLASS, /*!< This is a classical UML class diagram for the selected class and its direct children and parents. The nodes contain the methods and member variables with their visibility. */ @@ -256,9 +258,11 @@ class PythonServiceHandler : virtual public LanguageServiceIf of a module. */ }; - model::PYName queryNode(const std::string& id); + model::PYName queryNodeByID(const std::string& id); model::PYName queryNodeByPosition(const core::FilePosition& fpos); std::vector queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); + std::vector queryNodesInFile(const core::FileId& fileId); + std::vector queryNodes(const odb::query& odb_query); std::string getNodeLineValue(const model::PYName& pyname); private: diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 62521372c..a3ad98a14 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -29,7 +29,7 @@ void PythonServiceHandler::getAstNodeInfo( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); PythonServiceHandler::setInfoProperties(return_, pyname); return; @@ -50,7 +50,7 @@ void PythonServiceHandler::getSourceText( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); core::ProjectServiceHandler projectService(_db, _datadir, _context); std::string content; @@ -81,7 +81,7 @@ void PythonServiceHandler::getProperties( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); if(!pyname.full_name.empty()) { @@ -105,7 +105,7 @@ void PythonServiceHandler::getDiagramTypes( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); if(pyname.is_definition == true && pyname.type == "function") { @@ -123,7 +123,7 @@ void PythonServiceHandler::getDiagram( LOG(info) << "[PYSERVICE] " << __func__; python::Diagram diagram(_db, _datadir, _context); - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId_); + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); util::Graph graph = [&]() { switch (diagramId_) @@ -154,6 +154,7 @@ void PythonServiceHandler::getFileDiagramTypes( const core::FileId& fileId_) { LOG(info) << "[PYSERVICE] " << __func__; + return_.emplace("Module dependency", MODULE_DEPENDENCY); return; } @@ -163,6 +164,22 @@ void PythonServiceHandler::getFileDiagram( const int32_t diagramId_) { LOG(info) << "[PYSERVICE] " << __func__; + python::Diagram diagram(_db, _datadir, _context); + + util::Graph graph = [&]() + { + switch (diagramId_) + { + case MODULE_DEPENDENCY: + return diagram.getModuleDiagram(fileId_); + default: + return util::Graph(); + } + }(); + + if (graph.nodeCount() != 0) + return_ = graph.output(util::Graph::SVG); + return; } @@ -185,7 +202,7 @@ void PythonServiceHandler::getReferenceTypes( return_.emplace("Parameters", PARAMETER); return_.emplace("Caller", CALLER); - model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); + model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId); if(pyname.type == "function" && pyname.is_definition) { @@ -295,7 +312,7 @@ void PythonServiceHandler::getSyntaxHighlight( return; } -model::PYName PythonServiceHandler::queryNode(const std::string& id) +model::PYName PythonServiceHandler::queryNodeByID(const std::string& id) { return _transaction([&]() { @@ -334,12 +351,23 @@ model::PYName PythonServiceHandler::queryNodeByPosition(const core::FilePosition }); } +std::vector PythonServiceHandler::queryNodes(const odb::query& odb_query) +{ + return _transaction([&](){ + const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; + + odb::result nodes = _db->query(odb_query); + + return std::vector(nodes.begin(), nodes.end()); + }); +} + std::vector PythonServiceHandler::queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId) { return _transaction([&](){ odb::result nodes; - const model::PYName pyname = PythonServiceHandler::queryNode(astNodeId); + const model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId); const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; switch (referenceId) @@ -378,6 +406,16 @@ std::vector PythonServiceHandler::queryReferences(const core::Ast }); } +std::vector PythonServiceHandler::queryNodesInFile(const core::FileId& fileId) +{ + return _transaction([&](){ + const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; + odb::result nodes = _db->query((odb::query::file_id == std::stoull(fileId)) + order_by); + + return std::vector(nodes.begin(), nodes.end()); + }); +} + void PythonServiceHandler::setInfoProperties(AstNodeInfo& info, const model::PYName& pyname) { info.id = std::to_string(pyname.id); From f0058c52b0124d3224bb971ae88f232307cbfe1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 18 Sep 2024 15:13:39 +0200 Subject: [PATCH 074/156] [Diagrams] Refactor, module diagram --- plugins/python/service/src/diagram.cpp | 170 +++++++++++++++++-------- plugins/python/service/src/diagram.h | 17 ++- 2 files changed, 130 insertions(+), 57 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 25f2aa01b..b5690f935 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -17,87 +17,134 @@ Diagram::Diagram( util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) { - const std::vector callers = functionGoto(m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::CALLER), PythonServiceHandler::PARENT_FUNCTION); - const std::vector calls = functionGoto(m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::THIS_CALLS), PythonServiceHandler::DEFINITION); - + // Query calls + const std::vector this_calls = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::THIS_CALLS); + std::vector this_calls_refs(this_calls.size()); + std::transform(this_calls.begin(), this_calls.end(), this_calls_refs.begin(), [](const model::PYName& e){return e.ref_id;}); + const std::vector calls = m_pythonService.queryNodes(odb::query::ref_id.in_range(this_calls_refs.begin(), this_calls_refs.end()) && odb::query::is_definition == true && odb::query::is_import == false); + + // Query callers + const std::vector function_callers = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::CALLER); + std::vector callers_parents(function_callers.size()); + std::transform(function_callers.begin(), function_callers.end(), callers_parents.begin(), [](const model::PYName& e){return e.parent_function;}); + const std::vector callers = m_pythonService.queryNodes(odb::query::id.in_range(callers_parents.begin(), callers_parents.end())); + + // Create graph util::Graph graph; graph.setAttribute("rankdir", "LR"); - util::Graph::Node centerNode = addNode(graph, pyname); + // Add center node + util::Graph::Node centerNode = addPYNameNode(graph, pyname, true); const std::string centerNodeID = std::to_string(pyname.id); decorateNode(graph, centerNode, FunctionCenterNode); + // Add calls with definitions + std::unordered_map addedNodes; for (const model::PYName& node : calls) { - util::Graph::Node callNode = addNode(graph, node); - if(node.is_definition && (node.type == "function" || node.type == "class")) - { - decorateNode(graph, callNode, FunctionCallDefinitionNode); - } - else + addFunctionNode(graph, centerNode, node, FunctionCallDefinitionNode); + addedNodes.emplace(node.ref_id, true); + } + + // Add calls with missing definitions + for(const model::PYName& node : this_calls) + { + if(addedNodes.find(node.ref_id) == addedNodes.end()) { - decorateNode(graph, callNode, FunctionCallNode); - graph.setNodeAttribute(callNode, "id", centerNodeID); + addFunctionNode(graph, centerNode, node, FunctionCallNode); } - - util::Graph::Edge edge = graph.createEdge(centerNode, callNode); } + addedNodes.clear(); + + // Add callers with definitions for (const model::PYName& node : callers) { - util::Graph::Node callerNode = addNode(graph, node); - if(node.is_definition && (node.type == "function" || node.type == "class")) - { - decorateNode(graph, callerNode, FunctionCallerDefinitionNode); - } - else + addFunctionNode(graph, centerNode, node, FunctionCallerDefinitionNode); + } + + // Add callers with missing definitions + for (const model::PYName& node : function_callers) + { + if(node.parent_function == node.id) { - decorateNode(graph, callerNode, FunctionCallerNode); + addFunctionNode(graph, centerNode, node, FunctionCallerNode); } - - util::Graph::Edge edge = graph.createEdge(callerNode, centerNode); } return graph; } -std::vector Diagram::functionGoto(const std::vector& functions, const PythonServiceHandler::ReferenceType& ref_type) +util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) { - std::vector calls; - std::unordered_map added; + util::Graph graph; + graph.setAttribute("rankdir", "LR"); - // Find reference for each function - // Example use case: Find definition for each function - for(const model::PYName& p : functions) - { - std::vector defs = m_pythonService.queryReferences(std::to_string(p.id), ref_type); - if(defs.size() == 1) + const std::uint64_t file_id = std::stoull(fileId); + core::FileInfo mainFileInfo; + m_projectService.getFileInfo(mainFileInfo, fileId); + util::Graph::Node centerNode = addFileNode(graph, mainFileInfo); + decorateNode(graph, centerNode, FilePathCenterNode); + + // Query nodes + const std::vector nodesInFile = m_pythonService.queryNodesInFile(fileId); + std::vector refs(nodesInFile.size()); + std::transform(nodesInFile.begin(), nodesInFile.end(), refs.begin(), [](const model::PYName& e){return e.ref_id;}); + + const std::vector importedDefinitions = m_pythonService.queryNodes( + odb::query::ref_id.in_range(refs.begin(), refs.end()) && odb::query::is_definition == true && odb::query::is_import == false && odb::query::file_id != file_id); + + std::unordered_map map; + for (const model::PYName& d : importedDefinitions) { - model::PYName d = defs[0]; - - if(d.value.empty()) - { - calls.push_back(p); - continue; - } - - if(added.count(d.id) == 0) - { - calls.push_back(d); - added.emplace(d.id, true); - } - }else{ - calls.push_back(p); + core::FileInfo fileInfo; + m_projectService.getFileInfo(fileInfo, std::to_string(d.file_id)); + + util::Graph::Node node = [&](){ + auto it = map.find(d.file_id); + + if (it == map.end()) + { + util::Graph::Node n = addFileNode(graph, fileInfo); + graph.createEdge(centerNode, n); + + map.emplace(d.file_id, n); + return n; + }else{ + return it->second; + } + }(); + decorateNode(graph, node, FilePathNode); + + util::Graph::Node definitionNode = addPYNameNode(graph, d, false); + decorateNode(graph, definitionNode, ImportedNode); + graph.createEdge(node, definitionNode); } - } - return calls; + return graph; } -util::Graph::Node Diagram::addNode(util::Graph& graph_, const model::PYName& pyname) +void Diagram::addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType) +{ + util::Graph::Node node = addPYNameNode(graph_, pyname, true); + decorateNode(graph_, node, nodeType); + + if(nodeType == FunctionCallNode || nodeType == FunctionCallDefinitionNode) + { + graph_.createEdge(centerNode, node); + } + else + { + graph_.createEdge(node, centerNode); + } +} + +util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph) { const util::Graph::Subgraph subgraph = [&]() { + if(!addSubgraph) return util::Graph::Subgraph(); + const core::FileId fileId = std::to_string(pyname.file_id); auto it = m_subgraphs.find(fileId); @@ -136,11 +183,19 @@ util::Graph::Node Diagram::addNode(util::Graph& graph_, const model::PYName& pyn return node; } -void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeDecoration& decoration) +util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo) +{ + util::Graph::Node node = graph_.getOrCreateNode(fileInfo.id); + graph_.setNodeAttribute(node, "label", fileInfo.path); + + return node; +} + +void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeType& nodeType) { graph_.setNodeAttribute(node_, "style", "filled"); - switch(decoration) + switch(nodeType) { case FunctionCenterNode: graph_.setNodeAttribute(node_, "fillcolor", "gold"); @@ -159,6 +214,17 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const case FunctionCallDefinitionNode: graph_.setNodeAttribute(node_, "fillcolor", "lightblue"); break; + case FilePathNode: + graph_.setNodeAttribute(node_, "fillcolor", "limegreen"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; + case FilePathCenterNode: + graph_.setNodeAttribute(node_, "fillcolor", "gold"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; + case ImportedNode: + graph_.setNodeAttribute(node_, "fillcolor", "lightseagreen"); + break; } } } // python diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 09ee5c645..97a9bd825 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include namespace cc { @@ -23,21 +25,26 @@ namespace python const cc::webserver::ServerContext& context_); util::Graph getFunctionCallDiagram(const model::PYName& pyname); + util::Graph getModuleDiagram(const core::FileId& fileId); - enum NodeDecoration { + enum NodeType { FunctionCenterNode, FunctionCallNode, FunctionCallDefinitionNode, FunctionCallerNode, - FunctionCallerDefinitionNode + FunctionCallerDefinitionNode, + FilePathNode, + FilePathCenterNode, + ImportedNode }; private: PythonServiceHandler m_pythonService; core::ProjectServiceHandler m_projectService; std::map m_subgraphs; - std::vector functionGoto(const std::vector& functions, const PythonServiceHandler::ReferenceType& ref_type); - void decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeDecoration& decoration); - util::Graph::Node addNode(util::Graph& graph_, const model::PYName& pyname); + void addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType); + void decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeType& nodeType); + util::Graph::Node addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph); + util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo); }; } // python } // language From ed018d3ea4c9d85ca953129a6d8203c109c7fe22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 18 Sep 2024 15:35:39 +0200 Subject: [PATCH 075/156] [PyParser] Set a value for entire module nodes --- plugins/python/parser/pyparser/pyname.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index d34219cab..f698353b2 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -51,6 +51,11 @@ def getNamePosInfo(self) -> PosInfo: else: pos.value = self.name.get_line_code()[pos.column_start:] + if (self.name.module_path and + pos.line_start == 0 and pos.line_end == 0 and + pos.column_start == 0 and pos.column_end == 0): + pos.value = str(self.name.module_path).split("/")[-1] + return pos def getNodeInfo(self): From 0f64b3cb6f62cd7b31d0580480bd5f09a16fc8f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 18 Sep 2024 17:03:45 +0200 Subject: [PATCH 076/156] [Diagrams] Catch invalid file id exception --- plugins/python/service/src/diagram.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index b5690f935..f2c4eacfa 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -98,7 +98,11 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) for (const model::PYName& d : importedDefinitions) { core::FileInfo fileInfo; - m_projectService.getFileInfo(fileInfo, std::to_string(d.file_id)); + try { + m_projectService.getFileInfo(fileInfo, std::to_string(d.file_id)); + } catch (core::InvalidId) { + continue; + } util::Graph::Node node = [&](){ auto it = map.find(d.file_id); @@ -152,7 +156,11 @@ util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYNam return it->second; core::FileInfo fileInfo; - m_projectService.getFileInfo(fileInfo, fileId); + try { + m_projectService.getFileInfo(fileInfo, fileId); + } catch (core::InvalidId) { + return util::Graph::Subgraph(); + } util::Graph::Subgraph subgraph = graph_.getOrCreateSubgraph("cluster_" + fileInfo.path); From a85cb4694ff4b601fb4e8484e652dcd4929a464f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 19 Sep 2024 17:18:41 +0200 Subject: [PATCH 077/156] [Diagrams] Remove module nodes --- plugins/python/service/src/diagram.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index f2c4eacfa..370881fd1 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -110,6 +110,7 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) if (it == map.end()) { util::Graph::Node n = addFileNode(graph, fileInfo); + decorateNode(graph, n, FilePathNode); graph.createEdge(centerNode, n); map.emplace(d.file_id, n); @@ -118,7 +119,12 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) return it->second; } }(); - decorateNode(graph, node, FilePathNode); + + if(d.line_start == 0 && d.line_end == 0 && + d.column_start == 1 && d.column_end == 1) + { + continue; + } util::Graph::Node definitionNode = addPYNameNode(graph, d, false); decorateNode(graph, definitionNode, ImportedNode); From ea682895cefeb07ebb55f103d46c5e9e1e03d336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 20 Sep 2024 17:29:01 +0200 Subject: [PATCH 078/156] [Webgui] Handle AstNodes in a file diagram --- plugins/python/webgui/js/pythonDiagram.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/plugins/python/webgui/js/pythonDiagram.js b/plugins/python/webgui/js/pythonDiagram.js index a337ce99e..5ab698a8c 100644 --- a/plugins/python/webgui/js/pythonDiagram.js +++ b/plugins/python/webgui/js/pythonDiagram.js @@ -44,6 +44,12 @@ function (topic, Menu, MenuItem, PopupMenuItem, model, viewHandler) { id : 'python-file-diagram-handler', getDiagram : function (diagramType, nodeId, callback) { + var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); + + if(nodeInfo.id != 0) { + nodeId = nodeInfo.range.file; + } + model.pythonservice.getFileDiagram(nodeId, diagramType, callback); }, @@ -52,6 +58,21 @@ function (topic, Menu, MenuItem, PopupMenuItem, model, viewHandler) { }, mouseOverInfo : function (diagramType, nodeId) { + var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); + var range = nodeInfo.range.range; + + if(nodeInfo.id != 0) { + return { + fileId : nodeInfo.range.file, + selection : [ + range.startpos.line, + range.startpos.column, + range.endpos.line, + range.endpos.column + ] + }; + } + return { fileId : nodeId, selection : [1,1,1,1] From db09a70315a4b2b593c8cfaaf09f3a471d47458b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 20 Sep 2024 18:27:30 +0200 Subject: [PATCH 079/156] [PythonService] transformReferences, query definitions in file --- plugins/python/model/include/model/pyname.h | 7 +++++ .../service/include/service/pythonservice.h | 4 ++- plugins/python/service/src/diagram.cpp | 13 ++++----- plugins/python/service/src/diagram.h | 1 + plugins/python/service/src/pythonservice.cpp | 27 +++++++++++++++++-- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/plugins/python/model/include/model/pyname.h b/plugins/python/model/include/model/pyname.h index 1ef3e43de..54f8e99dd 100644 --- a/plugins/python/model/include/model/pyname.h +++ b/plugins/python/model/include/model/pyname.h @@ -10,6 +10,13 @@ namespace cc namespace model { +enum PYNameID { + ID, + REF_ID, + PARENT, + PARENT_FUNCTION +}; + #pragma db object struct PYName { diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 80e08b788..c9d139c46 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -1,6 +1,7 @@ #ifndef CC_SERVICE_PYTHON_PYTHONSERVICE_H #define CC_SERVICE_PYTHON_PYTHONSERVICE_H +#include #include #include #include @@ -261,8 +262,9 @@ class PythonServiceHandler : virtual public LanguageServiceIf model::PYName queryNodeByID(const std::string& id); model::PYName queryNodeByPosition(const core::FilePosition& fpos); std::vector queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); - std::vector queryNodesInFile(const core::FileId& fileId); + std::vector queryNodesInFile(const core::FileId& fileId, bool definitions); std::vector queryNodes(const odb::query& odb_query); + std::vector transformReferences(const std::vector& references, const model::PYNameID& id); std::string getNodeLineValue(const model::PYName& pyname); private: diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 370881fd1..0a5fb38f2 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -19,14 +19,12 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) { // Query calls const std::vector this_calls = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::THIS_CALLS); - std::vector this_calls_refs(this_calls.size()); - std::transform(this_calls.begin(), this_calls.end(), this_calls_refs.begin(), [](const model::PYName& e){return e.ref_id;}); + std::vector this_calls_refs = m_pythonService.transformReferences(this_calls, model::REF_ID); const std::vector calls = m_pythonService.queryNodes(odb::query::ref_id.in_range(this_calls_refs.begin(), this_calls_refs.end()) && odb::query::is_definition == true && odb::query::is_import == false); // Query callers const std::vector function_callers = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::CALLER); - std::vector callers_parents(function_callers.size()); - std::transform(function_callers.begin(), function_callers.end(), callers_parents.begin(), [](const model::PYName& e){return e.parent_function;}); + std::vector callers_parents = m_pythonService.transformReferences(function_callers, model::PARENT_FUNCTION); const std::vector callers = m_pythonService.queryNodes(odb::query::id.in_range(callers_parents.begin(), callers_parents.end())); // Create graph @@ -87,12 +85,11 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) decorateNode(graph, centerNode, FilePathCenterNode); // Query nodes - const std::vector nodesInFile = m_pythonService.queryNodesInFile(fileId); - std::vector refs(nodesInFile.size()); - std::transform(nodesInFile.begin(), nodesInFile.end(), refs.begin(), [](const model::PYName& e){return e.ref_id;}); + const std::vector nodesInFile = m_pythonService.transformReferences(m_pythonService.queryNodesInFile(fileId, false), model::REF_ID); const std::vector importedDefinitions = m_pythonService.queryNodes( - odb::query::ref_id.in_range(refs.begin(), refs.end()) && odb::query::is_definition == true && odb::query::is_import == false && odb::query::file_id != file_id); + odb::query::ref_id.in_range(nodesInFile.begin(), nodesInFile.end()) && odb::query::is_definition == true && odb::query::is_import == false && odb::query::file_id != file_id); + std::unordered_map map; for (const model::PYName& d : importedDefinitions) diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 97a9bd825..16150b0d5 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index a3ad98a14..6214cbd1f 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -406,16 +406,39 @@ std::vector PythonServiceHandler::queryReferences(const core::Ast }); } -std::vector PythonServiceHandler::queryNodesInFile(const core::FileId& fileId) +std::vector PythonServiceHandler::queryNodesInFile(const core::FileId& fileId, bool definitions) { return _transaction([&](){ const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; - odb::result nodes = _db->query((odb::query::file_id == std::stoull(fileId)) + order_by); + odb::result nodes = (definitions) ? + _db->query((odb::query::file_id == std::stoull(fileId) && odb::query::is_definition == true && odb::query::is_import == false) + order_by) + : _db->query((odb::query::file_id == std::stoull(fileId)) + order_by); return std::vector(nodes.begin(), nodes.end()); }); } +std::vector PythonServiceHandler::transformReferences(const std::vector& references, const model::PYNameID& id) +{ + std::vector ret(references.size()); + switch(id) + { + case model::ID: + std::transform(references.begin(), references.end(), ret.begin(), [](const model::PYName& e){return e.id;}); + break; + case model::REF_ID: + std::transform(references.begin(), references.end(), ret.begin(), [](const model::PYName& e){return e.ref_id;}); + break; + case model::PARENT: + std::transform(references.begin(), references.end(), ret.begin(), [](const model::PYName& e){return e.parent;}); + break; + case model::PARENT_FUNCTION: + std::transform(references.begin(), references.end(), ret.begin(), [](const model::PYName& e){return e.parent_function;}); + break; + } + return ret; +} + void PythonServiceHandler::setInfoProperties(AstNodeInfo& info, const model::PYName& pyname) { info.id = std::to_string(pyname.id); From a8c6374197941e63f268476955a3bae4926b57ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 20 Sep 2024 21:22:08 +0200 Subject: [PATCH 080/156] [Diagrams] Added imported usage to the module diagram --- plugins/python/service/src/diagram.cpp | 103 +++++++++++++++------- plugins/python/service/src/diagram.h | 7 +- plugins/python/webgui/js/pythonDiagram.js | 37 ++++---- 3 files changed, 96 insertions(+), 51 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 0a5fb38f2..2042029f7 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -86,48 +86,62 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) // Query nodes const std::vector nodesInFile = m_pythonService.transformReferences(m_pythonService.queryNodesInFile(fileId, false), model::REF_ID); + const std::vector definitionsInFile = m_pythonService.transformReferences(m_pythonService.queryNodesInFile(fileId, true), model::REF_ID); const std::vector importedDefinitions = m_pythonService.queryNodes( - odb::query::ref_id.in_range(nodesInFile.begin(), nodesInFile.end()) && odb::query::is_definition == true && odb::query::is_import == false && odb::query::file_id != file_id); + odb::query::ref_id.in_range(nodesInFile.begin(), nodesInFile.end()) && odb::query::is_definition == true && + odb::query::is_import == false && odb::query::file_id != file_id && + !(odb::query::line_start == 0 && odb::query::line_end == 0 && odb::query::column_start == 1 && odb::query::column_end == 1)); + const std::vector importedUsages = m_pythonService.queryNodes( + odb::query::ref_id.in_range(definitionsInFile.begin(), definitionsInFile.end()) && odb::query::is_definition == false && + odb::query::file_id != file_id); std::unordered_map map; - for (const model::PYName& d : importedDefinitions) + + auto getFileNode = [&](const model::PYName& p, const NodeType& nodeType) { core::FileInfo fileInfo; try { - m_projectService.getFileInfo(fileInfo, std::to_string(d.file_id)); + m_projectService.getFileInfo(fileInfo, std::to_string(p.file_id)); } catch (core::InvalidId) { - continue; + return util::Graph::Node(); } - util::Graph::Node node = [&](){ - auto it = map.find(d.file_id); - - if (it == map.end()) - { - util::Graph::Node n = addFileNode(graph, fileInfo); - decorateNode(graph, n, FilePathNode); - graph.createEdge(centerNode, n); + auto it = map.find(p.file_id); - map.emplace(d.file_id, n); - return n; - }else{ - return it->second; - } - }(); - - if(d.line_start == 0 && d.line_end == 0 && - d.column_start == 1 && d.column_end == 1) + if (it == map.end()) { - continue; + util::Graph::Node n = addFileNode(graph, fileInfo, centerNode, nodeType); + + map.emplace(p.file_id, n); + return n; + }else{ + return it->second; } + }; - util::Graph::Node definitionNode = addPYNameNode(graph, d, false); - decorateNode(graph, definitionNode, ImportedNode); - graph.createEdge(node, definitionNode); + for (const model::PYName& p : importedDefinitions) + { + util::Graph::Node node = getFileNode(p, ImportedFilePathNode); + if (node.empty()) continue; + + util::Graph::Node graphNode = addPYNameNode(graph, p, false); + decorateNode(graph, graphNode, ImportedNode); + graph.createEdge(node, graphNode); } + map.clear(); + + for (const model::PYName& p : importedUsages) + { + util::Graph::Node node = getFileNode(p, ImportsFilePathNode); + if (node.empty()) continue; + + util::Graph::Node graphNode = addPYNameNode(graph, p, false); + decorateNode(graph, graphNode, ImportsNode); + graph.createEdge(graphNode, node); + } return graph; } @@ -196,8 +210,25 @@ util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYNam util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo) { - util::Graph::Node node = graph_.getOrCreateNode(fileInfo.id); + util::Graph::Node node = graph_.getOrCreateNode("f" + fileInfo.id); + graph_.setNodeAttribute(node, "label", fileInfo.path); + + return node; +} + +util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType) +{ + const std::string id = (nodeType == ImportedFilePathNode) ? "d" + fileInfo.id : "s" + fileInfo.id; + + util::Graph::Node node = graph_.getOrCreateNode(id); graph_.setNodeAttribute(node, "label", fileInfo.path); + decorateNode(graph_, node, nodeType); + + if (nodeType == ImportedFilePathNode) { + graph_.createEdge(centerNode, node); + } else { + graph_.createEdge(node, centerNode); + } return node; } @@ -213,29 +244,37 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const break; case FunctionCallerNode: graph_.setNodeAttribute(node_, "fillcolor", "orange"); - graph_.setNodeAttribute(node_, "shape", "box"); + graph_.setNodeAttribute(node_, "shape", "cds"); break; case FunctionCallerDefinitionNode: graph_.setNodeAttribute(node_, "fillcolor", "coral"); break; case FunctionCallNode: graph_.setNodeAttribute(node_, "fillcolor", "cyan"); - graph_.setNodeAttribute(node_, "shape", "box"); + graph_.setNodeAttribute(node_, "shape", "cds"); break; case FunctionCallDefinitionNode: graph_.setNodeAttribute(node_, "fillcolor", "lightblue"); break; - case FilePathNode: - graph_.setNodeAttribute(node_, "fillcolor", "limegreen"); - graph_.setNodeAttribute(node_, "shape", "box"); - break; case FilePathCenterNode: graph_.setNodeAttribute(node_, "fillcolor", "gold"); graph_.setNodeAttribute(node_, "shape", "box"); break; + case ImportedFilePathNode: + graph_.setNodeAttribute(node_, "fillcolor", "limegreen"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; case ImportedNode: graph_.setNodeAttribute(node_, "fillcolor", "lightseagreen"); break; + case ImportsFilePathNode: + graph_.setNodeAttribute(node_, "fillcolor", "orange"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; + case ImportsNode: + graph_.setNodeAttribute(node_, "fillcolor", "coral"); + graph_.setNodeAttribute(node_, "shape", "cds"); + break; } } } // python diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 16150b0d5..60e6f5d47 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -34,9 +34,11 @@ namespace python FunctionCallDefinitionNode, FunctionCallerNode, FunctionCallerDefinitionNode, - FilePathNode, FilePathCenterNode, - ImportedNode + ImportedFilePathNode, + ImportsFilePathNode, + ImportedNode, + ImportsNode }; private: PythonServiceHandler m_pythonService; @@ -45,6 +47,7 @@ namespace python void addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType); void decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeType& nodeType); util::Graph::Node addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph); + util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType); util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo); }; } // python diff --git a/plugins/python/webgui/js/pythonDiagram.js b/plugins/python/webgui/js/pythonDiagram.js index 5ab698a8c..9158adb0e 100644 --- a/plugins/python/webgui/js/pythonDiagram.js +++ b/plugins/python/webgui/js/pythonDiagram.js @@ -44,9 +44,11 @@ function (topic, Menu, MenuItem, PopupMenuItem, model, viewHandler) { id : 'python-file-diagram-handler', getDiagram : function (diagramType, nodeId, callback) { - var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); - - if(nodeInfo.id != 0) { + // File path node + if (["d","s","f"].includes(nodeId[0])) { + nodeId = nodeId.substring(1); + } else { + var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); nodeId = nodeInfo.range.file; } @@ -58,24 +60,25 @@ function (topic, Menu, MenuItem, PopupMenuItem, model, viewHandler) { }, mouseOverInfo : function (diagramType, nodeId) { - var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); - var range = nodeInfo.range.range; - - if(nodeInfo.id != 0) { + // File path node + if (["d","s","f"].includes(nodeId[0])) { return { - fileId : nodeInfo.range.file, - selection : [ - range.startpos.line, - range.startpos.column, - range.endpos.line, - range.endpos.column - ] + fileId : nodeId.substring(1), + selection : [1,1,1,1] }; } + var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); + var range = nodeInfo.range.range; + return { - fileId : nodeId, - selection : [1,1,1,1] + fileId : nodeInfo.range.file, + selection : [ + range.startpos.line, + range.startpos.column, + range.endpos.line, + range.endpos.column + ] }; } }; @@ -102,7 +105,7 @@ function (topic, Menu, MenuItem, PopupMenuItem, model, viewHandler) { topic.publish('codecompass/openDiagram', { handler : 'python-file-diagram-handler', diagramType : diagramTypes[that.type], - node : fileInfo.id + node : "f" + fileInfo.id }); } })); From a91756c1881846c2a7c66c94c695eb10a93e3310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Fri, 20 Sep 2024 21:47:07 +0200 Subject: [PATCH 081/156] [Diagrams] Remove duplicate line nodes from module diagram --- plugins/python/service/src/diagram.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 2042029f7..0577d8b3f 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -133,14 +133,19 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) map.clear(); + std::unordered_map sameLine; for (const model::PYName& p : importedUsages) { util::Graph::Node node = getFileNode(p, ImportsFilePathNode); if (node.empty()) continue; + if (p.line_start == p.line_end && sameLine.find(p.line_start) != sameLine.end()) continue; + util::Graph::Node graphNode = addPYNameNode(graph, p, false); decorateNode(graph, graphNode, ImportsNode); graph.createEdge(graphNode, node); + + sameLine.emplace(p.line_start, true); } return graph; } From 226d59087c2bfc80d60ce72c64acc7d1b70e585a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 21 Sep 2024 11:19:44 +0000 Subject: [PATCH 082/156] [Diagrams] Usage diagrams --- .../service/include/service/pythonservice.h | 43 ++----------------- plugins/python/service/src/diagram.cpp | 31 +++++++++++-- plugins/python/service/src/diagram.h | 6 ++- plugins/python/service/src/pythonservice.cpp | 13 ++++-- 4 files changed, 45 insertions(+), 48 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index c9d139c46..c9ba4a197 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -216,47 +216,10 @@ class PythonServiceHandler : virtual public LanguageServiceIf enum DiagramType { - FUNCTION_CALL, /*!< In the function call diagram the nodes are functions and - the edges are the function calls between them. The diagram also displays - some dynamic information such as virtual function calls. */ - + FUNCTION_CALL, MODULE_DEPENDENCY, - - DETAILED_CLASS, /*!< This is a classical UML class diagram for the selected - class and its direct children and parents. The nodes contain the methods - and member variables with their visibility. */ - - CLASS_OVERVIEW, /*!< This is a class diagram which contains all classes - which inherit from the current one, and all parents from which the - current one inherits. The methods and member variables are node included - in the nodes, but the type of the member variables are indicated as - aggregation relationship. */ - - CLASS_COLLABORATION, /*!< This returns a class collaboration diagram - which shows the individual class members and their inheritance - hierarchy. */ - - COMPONENT_USERS, /*!< Component users diagram for source file S shows which - source files depend on S through the interfaces S provides. */ - - EXTERNAL_DEPENDENCY, /*!< This diagram shows the module which directory - depends on. The "depends on" diagram on module A traverses the - subdirectories of module A and shows all directories that contain files - that any of the source files in A includes. */ - - EXTERNAL_USERS, /*!< This diagram shows directories (modules) that are - users of the queried module. */ - - INCLUDE_DEPENDENCY, /*!< This diagram shows of the `#include` file - dependencies. */ - - INTERFACE, /*!< Interface diagram shows the used and provided interfaces of - a source code file and shows linking information. */ - - SUBSYSTEM_DEPENDENCY, /*!< This diagram shows the directories relationship - between the subdirectories of the queried module. This diagram is useful - to understand the relationships of the subdirectories (submodules) - of a module. */ + FUNCTION_USAGE, + CLASS_USAGE }; model::PYName queryNodeByID(const std::string& id); diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 0577d8b3f..96b6d4849 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -34,7 +34,7 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) // Add center node util::Graph::Node centerNode = addPYNameNode(graph, pyname, true); const std::string centerNodeID = std::to_string(pyname.id); - decorateNode(graph, centerNode, FunctionCenterNode); + decorateNode(graph, centerNode, CenterNode); // Add calls with definitions std::unordered_map addedNodes; @@ -150,6 +150,27 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) return graph; } +util::Graph Diagram::getUsageDiagram(const model::PYName& pyname) +{ + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + util::Graph::Node centerNode = addPYNameNode(graph, pyname, true); + decorateNode(graph, centerNode, CenterNode); + + if (!pyname.is_definition || pyname.is_import == true) return graph; + + const std::vector usages = m_pythonService.queryNodes( + odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false); + + for (const model::PYName& p : usages) + { + util::Graph::Node graphNode = addPYNameNode(graph, p, true); + decorateNode(graph, graphNode, p.is_call ? FunctionCallNode : UsageNode); + graph.createEdge(graphNode, centerNode); + } + return graph; +} + void Diagram::addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType) { util::Graph::Node node = addPYNameNode(graph_, pyname, true); @@ -244,7 +265,7 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const switch(nodeType) { - case FunctionCenterNode: + case CenterNode: graph_.setNodeAttribute(node_, "fillcolor", "gold"); break; case FunctionCallerNode: @@ -255,7 +276,7 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const graph_.setNodeAttribute(node_, "fillcolor", "coral"); break; case FunctionCallNode: - graph_.setNodeAttribute(node_, "fillcolor", "cyan"); + graph_.setNodeAttribute(node_, "fillcolor", "deepskyblue"); graph_.setNodeAttribute(node_, "shape", "cds"); break; case FunctionCallDefinitionNode: @@ -280,6 +301,10 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const graph_.setNodeAttribute(node_, "fillcolor", "coral"); graph_.setNodeAttribute(node_, "shape", "cds"); break; + case UsageNode: + graph_.setNodeAttribute(node_, "fillcolor", "cyan"); + graph_.setNodeAttribute(node_, "shape", "cds"); + break; } } } // python diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 60e6f5d47..427f59ea2 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -27,9 +27,10 @@ namespace python util::Graph getFunctionCallDiagram(const model::PYName& pyname); util::Graph getModuleDiagram(const core::FileId& fileId); + util::Graph getUsageDiagram(const model::PYName& pyname); enum NodeType { - FunctionCenterNode, + CenterNode, FunctionCallNode, FunctionCallDefinitionNode, FunctionCallerNode, @@ -38,7 +39,8 @@ namespace python ImportedFilePathNode, ImportsFilePathNode, ImportedNode, - ImportsNode + ImportsNode, + UsageNode }; private: PythonServiceHandler m_pythonService; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 6214cbd1f..03a6a4163 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -107,9 +107,13 @@ void PythonServiceHandler::getDiagramTypes( LOG(info) << "[PYSERVICE] " << __func__; model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); - if(pyname.is_definition == true && pyname.type == "function") - { - return_.emplace("Function call", FUNCTION_CALL); + if(pyname.is_definition == true && pyname.is_import == false) { + if(pyname.type == "function") { + return_.emplace("Function call", FUNCTION_CALL); + return_.emplace("Function usage", FUNCTION_USAGE); + } else if (pyname.type == "class") { + return_.emplace("Class Usage", CLASS_USAGE); + } } return; @@ -130,6 +134,9 @@ void PythonServiceHandler::getDiagram( { case FUNCTION_CALL: return diagram.getFunctionCallDiagram(pyname); + case FUNCTION_USAGE: + case CLASS_USAGE: + return diagram.getUsageDiagram(pyname); default: return util::Graph(); } From 3dc0fa1d5f78d0155f8576fb30652babda8335a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 21 Sep 2024 12:15:11 +0000 Subject: [PATCH 083/156] [PythonService] Node not found exceptions --- plugins/python/service/src/diagram.cpp | 4 ++-- plugins/python/service/src/pythonservice.cpp | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 96b6d4849..9ca91f5dd 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -104,7 +104,7 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) core::FileInfo fileInfo; try { m_projectService.getFileInfo(fileInfo, std::to_string(p.file_id)); - } catch (core::InvalidId) { + } catch (const core::InvalidId&) { return util::Graph::Node(); } @@ -201,7 +201,7 @@ util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYNam core::FileInfo fileInfo; try { m_projectService.getFileInfo(fileInfo, fileId); - } catch (core::InvalidId) { + } catch (const core::InvalidId&) { return util::Graph::Subgraph(); } diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 03a6a4163..99427cfd9 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -329,6 +329,11 @@ model::PYName PythonServiceHandler::queryNodeByID(const std::string& id) if(!nodes.empty()) { pyname = *nodes.begin(); + }else{ + LOG(info) << "[PYSERVICE] Node not found! (id = " << id << ")"; + core::InvalidId ex; + ex.__set_msg("Node not found!"); + throw ex; } return pyname; @@ -352,6 +357,9 @@ model::PYName PythonServiceHandler::queryNodeByPosition(const core::FilePosition pyname = *nodes.begin(); }else{ LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos.pos.line << " column = " << fpos.pos.column << ")"; + core::InvalidInput ex; + ex.__set_msg("Node not found!"); + throw ex; } return pyname; From 5c156ca1531a310ed605c2a67734844d24ea40ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 21 Sep 2024 15:00:43 +0000 Subject: [PATCH 084/156] [PythonParser] Improve database insert --- plugins/python/parser/src/pythonparser.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 5d244be4e..eb4023565 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -76,13 +76,13 @@ void PythonParser::parseProject(const std::string& root_path) // Insert into database LOG(info) << "[pythonparser] Inserting PYNames to database..."; - for(const auto& [key, value] : map) + cc::util::OdbTransaction {_ctx.db} ([&] { - cc::util::OdbTransaction {_ctx.db} ([&] + for(const auto& e : map) { - _ctx.db->persist(value); - }); - } + _ctx.db->persist(e.second); + } + }); LOG(info) << "[pythonparser] Parsing finished!"; LOG(info) << "[pythonparser] Inserted rows: " << map.size(); From 0c4b0735fe75bc295471963ad8f0326bc3c1deef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 21 Sep 2024 21:33:13 +0000 Subject: [PATCH 085/156] [PyParser] Make type_hint a config option --- plugins/python/parser/pyparser/parser.py | 10 ++++++++-- plugins/python/parser/pyparser/pyname.py | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 6e7872a74..921c4e421 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -11,6 +11,7 @@ def parseProject(root_path, venv_path, sys_path, n_proc): config = { "debug": True, "safe_env": False, + "type_hint": False, "root_path": root_path, "venv_path": None, "project": None @@ -24,6 +25,11 @@ def parseProject(root_path, venv_path, sys_path, n_proc): if not config["safe_env"]: log(f"{bcolors.WARNING}Creating Python environment in unsafe mode!") + if config["type_hint"]: + log(f"{bcolors.OKGREEN}Type hint support enabled!") + else: + log(f"{bcolors.OKBLUE}Type hint support disabled!") + try: if venv_path: jedi.create_environment(venv_path, safe = config["safe_env"]) @@ -82,10 +88,10 @@ def parse(path, config): for x in script.get_names(references = True, all_scopes = True): defs = x.goto(follow_imports = True, follow_builtin_imports = True) - putInMap(nodes, PYName(x).addDefs(defs, result).addASTHelper(asthelper).getNodeInfo()) + putInMap(nodes, PYName(x).addConfig(config).addDefs(defs, result).addASTHelper(asthelper).getNodeInfo()) for d in defs: - putInMap(nodes, PYName(d).getNodeInfo()) + putInMap(nodes, PYName(d).addConfig(config).getNodeInfo()) if d.module_path: result["imports"].append(str(d.module_path)) diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index f698353b2..34c435bab 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -19,6 +19,7 @@ def __init__(self, name: Name): self.refid = self.hashName self.defs = [] self.asthelper = None + self.config = None def addDefs(self, defs: List[Name], result): self.defs = defs @@ -34,6 +35,10 @@ def addASTHelper(self, asthelper: ASTHelper): self.asthelper = asthelper return self + def addConfig(self, config): + self.config = config + return self + def getNamePosInfo(self) -> PosInfo: pos = PosInfo() @@ -126,6 +131,9 @@ def __getParentFunction(self): def __getNameTypeHint(self): hint = "" + if not (self.config and self.config["type_hint"]): + return hint + try: res = self.name.get_type_hint() hint = res if res else "" From 34ef5dd9cf94ccd5e6bf2812dd699cd356304c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 21 Sep 2024 22:26:26 +0000 Subject: [PATCH 086/156] [PyParser] Make ASTHelper faster --- plugins/python/parser/pyparser/asthelper.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index f2b409a55..89f05d71d 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -4,26 +4,32 @@ class ASTHelper: astNodes: List[ast.AST] + calls: List[ast.Call] + imports: List[ast.Import | ast.ImportFrom] source: str def __init__(self, source): self.astNodes = [] + self.calls = [] + self.imports = [] self.source = source try: tree = ast.parse(source) self.astNodes = list(ast.walk(tree)) + self.calls = self.__getFunctionCalls() + self.imports = self.__getImports() except: pass - def getFunctionCalls(self) -> List[ast.Call]: + def __getFunctionCalls(self) -> List[ast.Call]: return cast(List[ast.Call], list(filter(lambda e : isinstance(e, ast.Call), self.astNodes))) - def getImports(self) -> List[ast.Import | ast.ImportFrom]: + def __getImports(self) -> List[ast.Import | ast.ImportFrom]: return cast(List[ast.Import | ast.ImportFrom], list(filter(lambda e : isinstance(e, ast.Import) or isinstance(e, ast.ImportFrom), self.astNodes))) def isFunctionCall(self, pos: PosInfo): - for e in self.getFunctionCalls(): + for e in self.calls: func = e.func if (isinstance(func, ast.Name)): @@ -48,7 +54,7 @@ def isFunctionCall(self, pos: PosInfo): return False def isImport(self, pos: PosInfo): - for e in self.getImports(): + for e in self.imports: if (e.lineno == pos.line_start and e.end_lineno == pos.line_end and e.col_offset == pos.column_start and From 6460c23023e9c6d7e48b850b8d7f1583b7ddc6db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 26 Sep 2024 15:01:18 +0200 Subject: [PATCH 087/156] [PyParser] Only process definitions once --- plugins/python/parser/pyparser/parser.py | 9 ++++----- plugins/python/parser/pyparser/pyname.py | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 921c4e421..5a24e4edc 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -75,7 +75,7 @@ def parse(path, config): "imports": [] } - nodes = {} + nodes: dict[int, PYName] = {} with open(path) as f: try: @@ -87,14 +87,13 @@ def parse(path, config): for x in script.get_names(references = True, all_scopes = True): defs = x.goto(follow_imports = True, follow_builtin_imports = True) + defs = list(map(lambda e : PYName(e), defs)) putInMap(nodes, PYName(x).addConfig(config).addDefs(defs, result).addASTHelper(asthelper).getNodeInfo()) for d in defs: - putInMap(nodes, PYName(d).addConfig(config).getNodeInfo()) - - if d.module_path: - result["imports"].append(str(d.module_path)) + if not (d.hashName in nodes): + putInMap(nodes, d.addConfig(config).appendModulePath(result["imports"]).getNodeInfo()) except: log(f"{bcolors.FAIL}Failed to parse file: {path}") diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 34c435bab..da37de5aa 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -10,7 +10,7 @@ class PYName: id: int name: Name refid: int - defs: List[Name] + defs: List['PYName'] asthelper: ASTHelper | None def __init__(self, name: Name): @@ -21,11 +21,11 @@ def __init__(self, name: Name): self.asthelper = None self.config = None - def addDefs(self, defs: List[Name], result): + def addDefs(self, defs: List['PYName'], result): self.defs = defs if len(defs) > 0: - self.refid = min(list(map(lambda e : PYName(e).hashName, defs))) + self.refid = min(list(map(lambda e : e.hashName, defs))) else: self.__reportMissingDefinition(result) @@ -39,6 +39,12 @@ def addConfig(self, config): self.config = config return self + def appendModulePath(self, import_list): + if self.name.module_path: + import_list.append(str(self.name.module_path)) + + return self + def getNamePosInfo(self) -> PosInfo: pos = PosInfo() @@ -68,7 +74,7 @@ def getNodeInfo(self): node["id"] = self.hashName node["ref_id"] = self.refid node["module_name"] = self.name.module_name - node["module_path"] = self.name.module_path + node["module_path"] = str(self.name.module_path) node["full_name"] = self.name.full_name if self.name.full_name else "" pos = self.getNamePosInfo() @@ -82,7 +88,7 @@ def getNodeInfo(self): node["is_definition"] = self.name.is_definition() node["file_id"] = self.__getFileId() node["type_hint"] = self.__getNameTypeHint() - node["is_builtin"] = self.name.in_builtin_module() or any(list(map(lambda x : x.in_builtin_module(), self.defs))) + node["is_builtin"] = self.name.in_builtin_module() or any(list(map(lambda x : x.name.in_builtin_module(), self.defs))) parent = self.name.parent() node["parent"] = PYName(parent).hashName if parent else node["id"] @@ -117,7 +123,7 @@ def __getParentFunction(self): try: node = self.name for _ in range(0,10): - parent: Name = node.parent() + parent: Name | None = node.parent() if parent and parent.type == "function" and parent.is_definition(): return PYName(parent).hashName elif parent: From 5ab718265293645eb4dc0cc631805c2735c893b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 30 Sep 2024 14:43:49 +0200 Subject: [PATCH 088/156] [PythonService] Debug build: ID, REF_ID, DEFINITION property --- plugins/python/service/src/pythonservice.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 99427cfd9..5e893bcba 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -97,6 +97,12 @@ void PythonServiceHandler::getProperties( return_.emplace("Function call", PythonServiceHandler::boolToString(pyname.is_call)); +#ifndef NDEBUG + return_.emplace("ID", std::to_string(pyname.id)); + return_.emplace("REF_ID", std::to_string(pyname.ref_id)); + return_.emplace("DEFINITION", PythonServiceHandler::boolToString(pyname.is_definition)); +#endif + return; } From 51e0535d432274629fe1a61288c41b268083b2ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 1 Oct 2024 13:39:33 +0200 Subject: [PATCH 089/156] [PyParser] Only calculate PosInfo once --- plugins/python/parser/pyparser/pyname.py | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index da37de5aa..565f71e8a 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -9,12 +9,14 @@ class PYName: id: int name: Name + pos: PosInfo refid: int defs: List['PYName'] asthelper: ASTHelper | None def __init__(self, name: Name): self.name = name + self.pos = self.__getNamePosInfo() self.hashName = self.__getHashName() self.refid = self.hashName self.defs = [] @@ -45,7 +47,7 @@ def appendModulePath(self, import_list): return self - def getNamePosInfo(self) -> PosInfo: + def __getNamePosInfo(self) -> PosInfo: pos = PosInfo() start_pos = self.name.get_definition_start_position() @@ -77,12 +79,11 @@ def getNodeInfo(self): node["module_path"] = str(self.name.module_path) node["full_name"] = self.name.full_name if self.name.full_name else "" - pos = self.getNamePosInfo() - node["line_start"] = pos.line_start - node["line_end"] = pos.line_end - node["column_start"] = pos.column_start + 1 - node["column_end"] = pos.column_end + 1 - node["value"] = pos.value + node["line_start"] = self.pos.line_start + node["line_end"] = self.pos.line_end + node["column_start"] = self.pos.column_start + 1 + node["column_end"] = self.pos.column_end + 1 + node["value"] = self.pos.value node["type"] = self.name.type node["is_definition"] = self.name.is_definition() @@ -97,16 +98,15 @@ def getNodeInfo(self): node["is_import"] = False if self.asthelper: - node["is_call"] = self.asthelper.isFunctionCall(pos) - node["is_import"] = self.asthelper.isImport(pos) + node["is_call"] = self.asthelper.isFunctionCall(self.pos) + node["is_import"] = self.asthelper.isImport(self.pos) node["parent_function"] = self.__getParentFunction() return node def __getHashName(self) -> int: - pos = self.getNamePosInfo() - s = f"{self.name.module_path}|{pos.line_start}|{pos.line_end}|{pos.column_start}|{pos.column_end}".encode("utf-8") + s = f"{self.name.module_path}|{self.pos.line_start}|{self.pos.line_end}|{self.pos.column_start}|{self.pos.column_end}".encode("utf-8") hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff return hash @@ -114,9 +114,8 @@ def __getFileId(self): return fnvHash(str(self.name.module_path)) def __reportMissingDefinition(self, result): - pos = self.getNamePosInfo() if not self.name.is_definition() and self.name.type == 'module': - log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.name.module_path} line = {pos.line_start})") + log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.name.module_path} line = {self.pos.line_start})") result["status"] = "partial" def __getParentFunction(self): From c8348be0df37a3f920e1dbcfadac25986957a902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 1 Oct 2024 15:07:25 +0200 Subject: [PATCH 090/156] [PyParser] PYReference, getFileRefs --- plugins/python/parser/pyparser/parser.py | 12 ++++-- plugins/python/parser/pyparser/pyname.py | 8 ++++ plugins/python/parser/pyparser/pyreference.py | 42 +++++++++++++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 plugins/python/parser/pyparser/pyreference.py diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 5a24e4edc..7d84f4d9c 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -6,12 +6,14 @@ from parserlog import log, bcolors from asthelper import ASTHelper from pyname import PYName +from pyreference import PYReference def parseProject(root_path, venv_path, sys_path, n_proc): config = { "debug": True, "safe_env": False, "type_hint": False, + "file_refs": True, "root_path": root_path, "venv_path": None, "project": None @@ -82,14 +84,16 @@ def parse(path, config): log(f"Parsing: {path}") source = f.read() script = jedi.Script(source, path=path, project=config["project"]) + names = script.get_names(references = True, all_scopes = True) asthelper = ASTHelper(source) + pyref = PYReference(config, script, names) - for x in script.get_names(references = True, all_scopes = True): - defs = x.goto(follow_imports = True, follow_builtin_imports = True) - defs = list(map(lambda e : PYName(e), defs)) + for x in names: + defs = pyref.getDefs(x) + refs = pyref.getFileRefs(x) - putInMap(nodes, PYName(x).addConfig(config).addDefs(defs, result).addASTHelper(asthelper).getNodeInfo()) + putInMap(nodes, PYName(x).addConfig(config).addDefs(defs, result).addRefs(refs).addASTHelper(asthelper).getNodeInfo()) for d in defs: if not (d.hashName in nodes): diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 565f71e8a..83930f8c0 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -33,6 +33,14 @@ def addDefs(self, defs: List['PYName'], result): return self + def addRefs(self, refs: List['PYName']): + defs = list(filter(lambda e : e.name.is_definition(), refs)) + + if len(defs) > 0: + self.refid = min(list(map(lambda e : e.hashName, defs))) + + return self + def addASTHelper(self, asthelper: ASTHelper): self.asthelper = asthelper return self diff --git a/plugins/python/parser/pyparser/pyreference.py b/plugins/python/parser/pyparser/pyreference.py new file mode 100644 index 000000000..f2b859de4 --- /dev/null +++ b/plugins/python/parser/pyparser/pyreference.py @@ -0,0 +1,42 @@ +from typing import List +from jedi import Script +from jedi.api.classes import Name +from pyname import PYName + +class PYReference: + script: Script + names: List[Name] + + def __init__(self, config, script: Script, names: List[Name]): + self.script = script + self.names = names + self.refmap = {} + + if config["file_refs"]: + self.__lookupFileRefs() + + def getDefs(self, name: Name) -> List[PYName]: + defs = name.goto(follow_imports = True, follow_builtin_imports = True) + defs = list(map(lambda e : PYName(e), defs)) + return defs + + def getFileRefs(self, name: Name) -> List[PYName]: + id = PYName(name).hashName + + if id in self.refmap: + return self.refmap[id] + else: + return [] + + def __lookupFileRefs(self): + for x in self.names: + if not(x.is_definition()): + continue + + refs = self.script.get_references(x.line, x.column, scope = "file") + refs = list(map(lambda e : PYName(e), refs)) + refs.append(PYName(x)) + + for r in refs: + self.refmap[r.hashName] = refs + From 9f2568957cc76e7ca1f979cb11159bd673c6c8a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 1 Oct 2024 16:38:07 +0200 Subject: [PATCH 091/156] [PythonService] queryNodeByPosition: prefer shorter node values --- plugins/python/service/src/pythonservice.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 5e893bcba..831ef5d9c 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -361,6 +361,14 @@ model::PYName PythonServiceHandler::queryNodeByPosition(const core::FilePosition if(!nodes.empty()) { pyname = *nodes.begin(); + + for(const model::PYName& p : nodes) + { + if(p.value.size() < pyname.value.size()) + { + pyname = p; + } + } }else{ LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos.pos.line << " column = " << fpos.pos.column << ")"; core::InvalidInput ex; From 74dd58b3b621475e5ab44dd01b66df85366792f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 1 Oct 2024 18:55:18 +0200 Subject: [PATCH 092/156] [PyParser] PYBuiltin --- plugins/python/parser/pyparser/parser.py | 3 +++ plugins/python/parser/pyparser/pybuiltin.py | 25 +++++++++++++++++++++ plugins/python/parser/pyparser/pyname.py | 3 ++- 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 plugins/python/parser/pyparser/pybuiltin.py diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 7d84f4d9c..77b598db5 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -7,6 +7,7 @@ from asthelper import ASTHelper from pyname import PYName from pyreference import PYReference +from pybuiltin import PYBuiltin def parseProject(root_path, venv_path, sys_path, n_proc): config = { @@ -77,6 +78,8 @@ def parse(path, config): "imports": [] } + PYBuiltin.findBuiltins() + nodes: dict[int, PYName] = {} with open(path) as f: diff --git a/plugins/python/parser/pyparser/pybuiltin.py b/plugins/python/parser/pyparser/pybuiltin.py new file mode 100644 index 000000000..9660be7eb --- /dev/null +++ b/plugins/python/parser/pyparser/pybuiltin.py @@ -0,0 +1,25 @@ +import sys +from jedi.api.classes import Name +from parserlog import log, bcolors + +class PYBuiltin: + builtin = {} + + @staticmethod + def findBuiltins(): + try: + # Note: Python 3.10+ required + stdlib_modules = sys.stdlib_module_names + for key, val in sys.modules.items(): + if hasattr(val, "__file__") and key in stdlib_modules: + PYBuiltin.builtin[val.__file__] = True + except: + log(f"{bcolors.FAIL}Failed to find Python builtins!") + + @staticmethod + def isBuiltin(name: Name): + path = str(name.module_path) + return (path in PYBuiltin.builtin or + name.in_builtin_module() or + "/typeshed/stdlib/" in path) + diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 83930f8c0..c419265c1 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -5,6 +5,7 @@ from parserlog import log, bcolors from asthelper import ASTHelper from posinfo import PosInfo +from pybuiltin import PYBuiltin class PYName: id: int @@ -97,7 +98,7 @@ def getNodeInfo(self): node["is_definition"] = self.name.is_definition() node["file_id"] = self.__getFileId() node["type_hint"] = self.__getNameTypeHint() - node["is_builtin"] = self.name.in_builtin_module() or any(list(map(lambda x : x.name.in_builtin_module(), self.defs))) + node["is_builtin"] = PYBuiltin.isBuiltin(self.name) or any(list(map(lambda x : PYBuiltin.isBuiltin(x.name), self.defs))) parent = self.name.parent() node["parent"] = PYName(parent).hashName if parent else node["id"] From ad63d877cc666c6a836a3753ff892eb995a90b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 1 Oct 2024 19:17:29 +0200 Subject: [PATCH 093/156] [Diagrams] Simplify module dependency diagram --- plugins/python/service/src/diagram.cpp | 14 -------------- plugins/python/service/src/diagram.h | 1 - 2 files changed, 15 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 9ca91f5dd..55269834a 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -133,19 +133,9 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) map.clear(); - std::unordered_map sameLine; for (const model::PYName& p : importedUsages) { util::Graph::Node node = getFileNode(p, ImportsFilePathNode); - if (node.empty()) continue; - - if (p.line_start == p.line_end && sameLine.find(p.line_start) != sameLine.end()) continue; - - util::Graph::Node graphNode = addPYNameNode(graph, p, false); - decorateNode(graph, graphNode, ImportsNode); - graph.createEdge(graphNode, node); - - sameLine.emplace(p.line_start, true); } return graph; } @@ -297,10 +287,6 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const graph_.setNodeAttribute(node_, "fillcolor", "orange"); graph_.setNodeAttribute(node_, "shape", "box"); break; - case ImportsNode: - graph_.setNodeAttribute(node_, "fillcolor", "coral"); - graph_.setNodeAttribute(node_, "shape", "cds"); - break; case UsageNode: graph_.setNodeAttribute(node_, "fillcolor", "cyan"); graph_.setNodeAttribute(node_, "shape", "cds"); diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 427f59ea2..9aa987d1b 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -39,7 +39,6 @@ namespace python ImportedFilePathNode, ImportsFilePathNode, ImportedNode, - ImportsNode, UsageNode }; private: From bd3c86200e700ce9c3fbbb3fc89732c5fa2f72b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 1 Oct 2024 20:03:35 +0200 Subject: [PATCH 094/156] [PythonService] Refactor getDiagramTypes --- plugins/python/service/src/pythonservice.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 831ef5d9c..01fcd2daf 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -111,11 +111,19 @@ void PythonServiceHandler::getDiagramTypes( const core::AstNodeId& astNodeId_) { LOG(info) << "[PYSERVICE] " << __func__; - model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); + const model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); + + if (pyname.is_import == true) return; - if(pyname.is_definition == true && pyname.is_import == false) { - if(pyname.type == "function") { + if (pyname.is_definition == true && pyname.type == "function") { return_.emplace("Function call", FUNCTION_CALL); + } + + // Usage diagrams + const size_t count = PythonServiceHandler::queryReferences(astNodeId_, USAGE).size(); + + if (count > 0 && pyname.is_definition == true) { + if (pyname.type == "function") { return_.emplace("Function usage", FUNCTION_USAGE); } else if (pyname.type == "class") { return_.emplace("Class Usage", CLASS_USAGE); From cd80bbb286f522a4c9a153fc91cab937c3f0a870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 2 Oct 2024 11:19:53 +0200 Subject: [PATCH 095/156] [PyParser] PosInfo dataclass --- plugins/python/parser/pyparser/posinfo.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/plugins/python/parser/pyparser/posinfo.py b/plugins/python/parser/pyparser/posinfo.py index 65b1c1635..7a22021f0 100644 --- a/plugins/python/parser/pyparser/posinfo.py +++ b/plugins/python/parser/pyparser/posinfo.py @@ -1,14 +1,10 @@ -class PosInfo: - line_start: int - line_end: int - column_start: int - column_end: int - value: str +from dataclasses import dataclass - def __init__(self): - self.line_start = 0 - self.line_end = 0 - self.column_start = 0 - self.column_end = 0 - self.value = "" +@dataclass +class PosInfo: + line_start: int = 0 + line_end: int = 0 + column_start: int = 0 + column_end: int = 0 + value: str = "" From b02fb175b353bd8327ee16140b077db0dd437c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 2 Oct 2024 11:44:29 +0200 Subject: [PATCH 096/156] [PyParser] ParserConfig --- plugins/python/parser/pyparser/parser.py | 36 ++++++++----------- .../python/parser/pyparser/parserconfig.py | 12 +++++++ plugins/python/parser/pyparser/pyname.py | 6 ++-- plugins/python/parser/pyparser/pyreference.py | 7 ++-- 4 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 plugins/python/parser/pyparser/parserconfig.py diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 77b598db5..3617886a6 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -6,37 +6,29 @@ from parserlog import log, bcolors from asthelper import ASTHelper from pyname import PYName +from parserconfig import ParserConfig from pyreference import PYReference from pybuiltin import PYBuiltin def parseProject(root_path, venv_path, sys_path, n_proc): - config = { - "debug": True, - "safe_env": False, - "type_hint": False, - "file_refs": True, - "root_path": root_path, - "venv_path": None, - "project": None - } - log(f"Parsing project: {root_path}") + config = ParserConfig(root_path=root_path) - if config["debug"]: + if config.debug: log(f"{bcolors.WARNING}Parsing in debug mode!") - if not config["safe_env"]: + if not config.safe_env: log(f"{bcolors.WARNING}Creating Python environment in unsafe mode!") - if config["type_hint"]: + if config.type_hint: log(f"{bcolors.OKGREEN}Type hint support enabled!") else: log(f"{bcolors.OKBLUE}Type hint support disabled!") try: if venv_path: - jedi.create_environment(venv_path, safe = config["safe_env"]) - config["venv_path"] = venv_path + jedi.create_environment(venv_path, safe = config.safe_env) + config.venv_path = venv_path log(f"{bcolors.OKGREEN}Using virtual environment: {venv_path}") else: venv_path = None @@ -44,11 +36,11 @@ def parseProject(root_path, venv_path, sys_path, n_proc): if sys_path: log(f"{bcolors.OKGREEN}Using additional syspath: {sys_path}") - config["project"] = jedi.Project(path = root_path, environment_path = venv_path, added_sys_path = sys_path) + config.project = jedi.Project(path = root_path, environment_path = venv_path, added_sys_path = sys_path) except: log(f"{bcolors.FAIL}Failed to use virtual environment: {venv_path}") - if config["debug"]: + if config.debug: traceback.print_exc() py_files = [] @@ -63,14 +55,14 @@ def parseProject(root_path, venv_path, sys_path, n_proc): if ext and ext.lower() == '.py': py_files.append(p) - if config["venv_path"]: - py_files = filter(lambda e : not(e.startswith(config["venv_path"])), py_files) + if config.venv_path: + py_files = filter(lambda e : not(e.startswith(config.venv_path)), py_files) with multiprocessing.Pool(processes=n_proc) as pool: results = pool.starmap(parse, zip(py_files, repeat(config))) return results -def parse(path, config): +def parse(path: str, config: ParserConfig): result = { "path": path, "status": "full", @@ -86,7 +78,7 @@ def parse(path, config): try: log(f"Parsing: {path}") source = f.read() - script = jedi.Script(source, path=path, project=config["project"]) + script = jedi.Script(source, path=path, project=config.project) names = script.get_names(references = True, all_scopes = True) asthelper = ASTHelper(source) @@ -104,7 +96,7 @@ def parse(path, config): except: log(f"{bcolors.FAIL}Failed to parse file: {path}") - if config["debug"]: + if config.debug: traceback.print_exc() result["nodes"] = list(nodes.values()) diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py new file mode 100644 index 000000000..834208ada --- /dev/null +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass +from jedi import Project + +@dataclass +class ParserConfig: + root_path: str + venv_path: str | None = None + project: Project | None = None + debug: bool = True + safe_env: bool = False + type_hint: bool = False + file_refs: bool = True diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index c419265c1..c85c254db 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -6,6 +6,7 @@ from asthelper import ASTHelper from posinfo import PosInfo from pybuiltin import PYBuiltin +from parserconfig import ParserConfig class PYName: id: int @@ -14,6 +15,7 @@ class PYName: refid: int defs: List['PYName'] asthelper: ASTHelper | None + config: ParserConfig | None def __init__(self, name: Name): self.name = name @@ -46,7 +48,7 @@ def addASTHelper(self, asthelper: ASTHelper): self.asthelper = asthelper return self - def addConfig(self, config): + def addConfig(self, config: ParserConfig): self.config = config return self @@ -145,7 +147,7 @@ def __getParentFunction(self): def __getNameTypeHint(self): hint = "" - if not (self.config and self.config["type_hint"]): + if not (self.config and self.config.type_hint): return hint try: diff --git a/plugins/python/parser/pyparser/pyreference.py b/plugins/python/parser/pyparser/pyreference.py index f2b859de4..13726ede8 100644 --- a/plugins/python/parser/pyparser/pyreference.py +++ b/plugins/python/parser/pyparser/pyreference.py @@ -2,17 +2,20 @@ from jedi import Script from jedi.api.classes import Name from pyname import PYName +from parserconfig import ParserConfig class PYReference: script: Script names: List[Name] + config: ParserConfig - def __init__(self, config, script: Script, names: List[Name]): + def __init__(self, config: ParserConfig, script: Script, names: List[Name]): + self.config = config self.script = script self.names = names self.refmap = {} - if config["file_refs"]: + if self.config.file_refs: self.__lookupFileRefs() def getDefs(self, name: Name) -> List[PYName]: From 6c27abac8533d65eaf18b197f288a20369bd29fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 2 Oct 2024 15:03:04 +0200 Subject: [PATCH 097/156] [PyParser] PYReference error handling, stack trace config option --- plugins/python/parser/pyparser/parser.py | 7 ++-- .../python/parser/pyparser/parserconfig.py | 1 + plugins/python/parser/pyparser/pyreference.py | 34 ++++++++++++++----- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 3617886a6..54d2efb43 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -25,6 +25,9 @@ def parseProject(root_path, venv_path, sys_path, n_proc): else: log(f"{bcolors.OKBLUE}Type hint support disabled!") + if config.stack_trace: + log(f"{bcolors.OKGREEN}Stack trace enabled!") + try: if venv_path: jedi.create_environment(venv_path, safe = config.safe_env) @@ -40,7 +43,7 @@ def parseProject(root_path, venv_path, sys_path, n_proc): except: log(f"{bcolors.FAIL}Failed to use virtual environment: {venv_path}") - if config.debug: + if config.stack_trace: traceback.print_exc() py_files = [] @@ -96,7 +99,7 @@ def parse(path: str, config: ParserConfig): except: log(f"{bcolors.FAIL}Failed to parse file: {path}") - if config.debug: + if config.stack_trace: traceback.print_exc() result["nodes"] = list(nodes.values()) diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py index 834208ada..94835a403 100644 --- a/plugins/python/parser/pyparser/parserconfig.py +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -7,6 +7,7 @@ class ParserConfig: venv_path: str | None = None project: Project | None = None debug: bool = True + stack_trace: bool = False safe_env: bool = False type_hint: bool = False file_refs: bool = True diff --git a/plugins/python/parser/pyparser/pyreference.py b/plugins/python/parser/pyparser/pyreference.py index 13726ede8..70f2ff289 100644 --- a/plugins/python/parser/pyparser/pyreference.py +++ b/plugins/python/parser/pyparser/pyreference.py @@ -3,6 +3,8 @@ from jedi.api.classes import Name from pyname import PYName from parserconfig import ParserConfig +from parserlog import log, bcolors +import traceback class PYReference: script: Script @@ -19,9 +21,17 @@ def __init__(self, config: ParserConfig, script: Script, names: List[Name]): self.__lookupFileRefs() def getDefs(self, name: Name) -> List[PYName]: - defs = name.goto(follow_imports = True, follow_builtin_imports = True) - defs = list(map(lambda e : PYName(e), defs)) - return defs + try: + defs = name.goto(follow_imports = True, follow_builtin_imports = True) + defs = list(map(lambda e : PYName(e), defs)) + return defs + except: + if self.config.debug: + log(f"{bcolors.FAIL}Failed to find definition! (file = {str(name.module_path)} line = {name.line} column = {name.column})") + if self.config.stack_trace: + traceback.print_exc() + + return [] def getFileRefs(self, name: Name) -> List[PYName]: id = PYName(name).hashName @@ -36,10 +46,16 @@ def __lookupFileRefs(self): if not(x.is_definition()): continue - refs = self.script.get_references(x.line, x.column, scope = "file") - refs = list(map(lambda e : PYName(e), refs)) - refs.append(PYName(x)) - - for r in refs: - self.refmap[r.hashName] = refs + try: + refs = self.script.get_references(x.line, x.column, scope = "file") + refs = list(map(lambda e : PYName(e), refs)) + refs.append(PYName(x)) + + for r in refs: + self.refmap[r.hashName] = refs + except: + if self.config.debug: + log(f"{bcolors.FAIL}Failed to find references! (file = {str(x.module_path)} line = {x.line} column = {x.column})") + if self.config.stack_trace: + traceback.print_exc() From d72b6ffa3d1bd4ee15bf330093bea571f8410dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 2 Oct 2024 15:12:24 +0200 Subject: [PATCH 098/156] [PyParser] module_path should be str --- plugins/python/parser/pyparser/pyname.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index c85c254db..ea42b4b22 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -117,7 +117,7 @@ def getNodeInfo(self): return node def __getHashName(self) -> int: - s = f"{self.name.module_path}|{self.pos.line_start}|{self.pos.line_end}|{self.pos.column_start}|{self.pos.column_end}".encode("utf-8") + s = f"{str(self.name.module_path)}|{self.pos.line_start}|{self.pos.line_end}|{self.pos.column_start}|{self.pos.column_end}".encode("utf-8") hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff return hash @@ -126,7 +126,7 @@ def __getFileId(self): def __reportMissingDefinition(self, result): if not self.name.is_definition() and self.name.type == 'module': - log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.name.module_path} line = {self.pos.line_start})") + log(f"{bcolors.FAIL}Missing {self.name.description} (file = {str(self.name.module_path)} line = {self.pos.line_start})") result["status"] = "partial" def __getParentFunction(self): From 9db120876f2bd5d42351cda8439046b6455cf33c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 2 Oct 2024 15:13:35 +0200 Subject: [PATCH 099/156] [PyParser] Stack trace warning color --- plugins/python/parser/pyparser/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 54d2efb43..eb7cb9fc8 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -26,7 +26,7 @@ def parseProject(root_path, venv_path, sys_path, n_proc): log(f"{bcolors.OKBLUE}Type hint support disabled!") if config.stack_trace: - log(f"{bcolors.OKGREEN}Stack trace enabled!") + log(f"{bcolors.WARNING}Stack trace enabled!") try: if venv_path: From 60ac1c1197e1ae634ec732548327cc7ea39d18fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 5 Oct 2024 17:42:30 +0200 Subject: [PATCH 100/156] [PyParser] PYBuiltin stack trace --- plugins/python/parser/pyparser/parser.py | 2 +- plugins/python/parser/pyparser/pybuiltin.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index eb7cb9fc8..796cab473 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -73,7 +73,7 @@ def parse(path: str, config: ParserConfig): "imports": [] } - PYBuiltin.findBuiltins() + PYBuiltin.findBuiltins(config) nodes: dict[int, PYName] = {} diff --git a/plugins/python/parser/pyparser/pybuiltin.py b/plugins/python/parser/pyparser/pybuiltin.py index 9660be7eb..6dca80c95 100644 --- a/plugins/python/parser/pyparser/pybuiltin.py +++ b/plugins/python/parser/pyparser/pybuiltin.py @@ -1,12 +1,14 @@ import sys +import traceback from jedi.api.classes import Name from parserlog import log, bcolors +from parserconfig import ParserConfig class PYBuiltin: builtin = {} @staticmethod - def findBuiltins(): + def findBuiltins(config: ParserConfig): try: # Note: Python 3.10+ required stdlib_modules = sys.stdlib_module_names @@ -15,6 +17,8 @@ def findBuiltins(): PYBuiltin.builtin[val.__file__] = True except: log(f"{bcolors.FAIL}Failed to find Python builtins!") + if config.stack_trace: + traceback.print_exc() @staticmethod def isBuiltin(name: Name): From 57ccf006311b47417dd999f82ac4f04dcf144249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 5 Oct 2024 18:13:11 +0200 Subject: [PATCH 101/156] [Diagrams] Use different color for builtin paths --- plugins/python/service/src/diagram.cpp | 12 +++++++++--- plugins/python/service/src/diagram.h | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 55269834a..d66ede201 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -123,7 +123,7 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) for (const model::PYName& p : importedDefinitions) { - util::Graph::Node node = getFileNode(p, ImportedFilePathNode); + util::Graph::Node node = getFileNode(p, p.is_builtin ? ImportedBuiltinFilePathNode : ImportedFilePathNode); if (node.empty()) continue; util::Graph::Node graphNode = addPYNameNode(graph, p, false); @@ -198,9 +198,11 @@ util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYNam util::Graph::Subgraph subgraph = graph_.getOrCreateSubgraph("cluster_" + fileInfo.path); + const std::string pathColor = pyname.is_builtin ? "dodgerblue" : "limegreen"; + graph_.setSubgraphAttribute(subgraph, "id", fileInfo.id); const std::string coloredLabel = - "
" + + "
" + fileInfo.path + "
"; graph_.setSubgraphAttribute(subgraph, "label", coloredLabel, true); @@ -240,7 +242,7 @@ util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo graph_.setNodeAttribute(node, "label", fileInfo.path); decorateNode(graph_, node, nodeType); - if (nodeType == ImportedFilePathNode) { + if (nodeType == ImportedFilePathNode || nodeType == ImportedBuiltinFilePathNode) { graph_.createEdge(centerNode, node); } else { graph_.createEdge(node, centerNode); @@ -280,6 +282,10 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const graph_.setNodeAttribute(node_, "fillcolor", "limegreen"); graph_.setNodeAttribute(node_, "shape", "box"); break; + case ImportedBuiltinFilePathNode: + graph_.setNodeAttribute(node_, "fillcolor", "dodgerblue"); + graph_.setNodeAttribute(node_, "shape", "box"); + break; case ImportedNode: graph_.setNodeAttribute(node_, "fillcolor", "lightseagreen"); break; diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 9aa987d1b..7ad66e761 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -37,6 +37,7 @@ namespace python FunctionCallerDefinitionNode, FilePathCenterNode, ImportedFilePathNode, + ImportedBuiltinFilePathNode, ImportsFilePathNode, ImportedNode, UsageNode From c725657acfe9ba2b8810160e57ce163245a62a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 5 Oct 2024 18:31:02 +0200 Subject: [PATCH 102/156] [PythonService] queryNodes ORDER BY is_builtin --- plugins/python/service/src/pythonservice.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 01fcd2daf..506a46e93 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -391,9 +391,10 @@ model::PYName PythonServiceHandler::queryNodeByPosition(const core::FilePosition std::vector PythonServiceHandler::queryNodes(const odb::query& odb_query) { return _transaction([&](){ - const odb::query order_by = "ORDER BY" + odb::query::line_start + "," + odb::query::column_start; + const odb::query order_by = "ORDER BY" + odb::query::is_builtin + "," + odb::query::line_start + "," + odb::query::column_start; + const odb::query q = odb_query + order_by; - odb::result nodes = _db->query(odb_query); + odb::result nodes = _db->query(q); return std::vector(nodes.begin(), nodes.end()); }); From 889bb585200e27bd417c4ecf84c38fac3d054668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 5 Oct 2024 19:46:36 +0200 Subject: [PATCH 103/156] [PyParser] PYBuiltin rework --- plugins/python/parser/pyparser/pybuiltin.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/python/parser/pyparser/pybuiltin.py b/plugins/python/parser/pyparser/pybuiltin.py index 6dca80c95..2114600ec 100644 --- a/plugins/python/parser/pyparser/pybuiltin.py +++ b/plugins/python/parser/pyparser/pybuiltin.py @@ -1,4 +1,5 @@ import sys +import importlib.util import traceback from jedi.api.classes import Name from parserlog import log, bcolors @@ -12,9 +13,12 @@ def findBuiltins(config: ParserConfig): try: # Note: Python 3.10+ required stdlib_modules = sys.stdlib_module_names - for key, val in sys.modules.items(): - if hasattr(val, "__file__") and key in stdlib_modules: - PYBuiltin.builtin[val.__file__] = True + + for e in stdlib_modules: + spec = importlib.util.find_spec(e) + if spec and spec.origin: + PYBuiltin.builtin[spec.origin] = True + except: log(f"{bcolors.FAIL}Failed to find Python builtins!") if config.stack_trace: From f08cffdef8c87c39d1a8878bec462e554527db4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 5 Oct 2024 19:58:13 +0200 Subject: [PATCH 104/156] [Diagrams] Fix file path color --- plugins/python/service/src/diagram.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index d66ede201..09aa48c34 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -198,7 +198,7 @@ util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYNam util::Graph::Subgraph subgraph = graph_.getOrCreateSubgraph("cluster_" + fileInfo.path); - const std::string pathColor = pyname.is_builtin ? "dodgerblue" : "limegreen"; + const std::string pathColor = (pyname.is_builtin && pyname.is_definition) ? "dodgerblue" : "limegreen"; graph_.setSubgraphAttribute(subgraph, "id", fileInfo.id); const std::string coloredLabel = From 908dba7c8b89cfc675a961c48a2522a264cc8081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 5 Oct 2024 20:26:57 +0200 Subject: [PATCH 105/156] [PyParser] PYBuiltin stdlib submodules --- plugins/python/parser/pyparser/pybuiltin.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/plugins/python/parser/pyparser/pybuiltin.py b/plugins/python/parser/pyparser/pybuiltin.py index 2114600ec..c16f7cae8 100644 --- a/plugins/python/parser/pyparser/pybuiltin.py +++ b/plugins/python/parser/pyparser/pybuiltin.py @@ -1,4 +1,5 @@ import sys +import os import importlib.util import traceback from jedi.api.classes import Name @@ -19,6 +20,16 @@ def findBuiltins(config: ParserConfig): if spec and spec.origin: PYBuiltin.builtin[spec.origin] = True + if spec and spec.submodule_search_locations: + for submodule_dir in spec.submodule_search_locations: + for root, dirs, files in os.walk(submodule_dir): + for file in files: + p = os.path.join(root, file) + ext = os.path.splitext(p)[1] + + if ext and ext.lower() == '.py': + PYBuiltin.builtin[p] = True + except: log(f"{bcolors.FAIL}Failed to find Python builtins!") if config.stack_trace: From 194bda76e949155e9cec685ae921a52790afca32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 5 Oct 2024 21:21:15 +0200 Subject: [PATCH 106/156] [Diagrams] Add a comment explanation to file path nodes --- plugins/python/service/src/diagram.cpp | 8 +++++++- plugins/python/webgui/js/pythonDiagram.js | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 09aa48c34..161184914 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -236,7 +236,13 @@ util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType) { - const std::string id = (nodeType == ImportedFilePathNode) ? "d" + fileInfo.id : "s" + fileInfo.id; + /* We might need to add a file path multiple times to the diagram. + Since we need unique ids, we will differentiate nodes based on starting character: + 'f' - regular file path node + 'd' - ImportedFilePathNode, ImportedBuiltinFilePathNode + 's' - ImportsFilePathNode + Any id without a {'f', 'd', 's'} starting character is not a file path node. */ + const std::string id = (nodeType == ImportedFilePathNode || nodeType == ImportedBuiltinFilePathNode) ? "d" + fileInfo.id : "s" + fileInfo.id; util::Graph::Node node = graph_.getOrCreateNode(id); graph_.setNodeAttribute(node, "label", fileInfo.path); diff --git a/plugins/python/webgui/js/pythonDiagram.js b/plugins/python/webgui/js/pythonDiagram.js index 9158adb0e..e0a16e15f 100644 --- a/plugins/python/webgui/js/pythonDiagram.js +++ b/plugins/python/webgui/js/pythonDiagram.js @@ -46,6 +46,8 @@ function (topic, Menu, MenuItem, PopupMenuItem, model, viewHandler) { getDiagram : function (diagramType, nodeId, callback) { // File path node if (["d","s","f"].includes(nodeId[0])) { + // When requesting a file diagram, a valid file needs to be used. + // The first character is therefore removed. nodeId = nodeId.substring(1); } else { var nodeInfo = model.pythonservice.getAstNodeInfo(nodeId); From a865b4b9ba51924001d1aa9ccde33dc494b156f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 5 Oct 2024 22:51:52 +0200 Subject: [PATCH 107/156] [Diagrams] Diagram Legend --- plugins/python/service/src/diagram.cpp | 59 ++++++++++++++++++++ plugins/python/service/src/diagram.h | 4 ++ plugins/python/service/src/pythonservice.cpp | 35 ++++++++++++ 3 files changed, 98 insertions(+) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 161184914..c58a576cd 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -305,6 +305,65 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const break; } } + +util::Graph Diagram::getFunctionCallDiagramLegend() +{ + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + addLegendNode(graph, ImportedBuiltinFilePathNode, "Builtin file path"); + addLegendNode(graph, ImportedFilePathNode, "File path"); + addLegendNode(graph, FunctionCallNode, "Function call without definition"); + addLegendNode(graph, FunctionCallDefinitionNode, "Function call"); + addLegendNode(graph, FunctionCallerNode, "Line of code calling selected node"); + addLegendNode(graph, FunctionCallerDefinitionNode, "Function calling selected node"); + addLegendNode(graph, CenterNode, "Selected node"); + + return graph; +} + +util::Graph Diagram::getUsageDiagramLegend() +{ + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + addLegendNode(graph, ImportedBuiltinFilePathNode, "Builtin module path"); + addLegendNode(graph, ImportedFilePathNode, "Module path"); + addLegendNode(graph, FunctionCallNode, "Function call"); + addLegendNode(graph, UsageNode, "Usage node"); + addLegendNode(graph, CenterNode, "Selected node"); + + return graph; +} + +util::Graph Diagram::getModuleDiagramLegend() +{ + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + addLegendNode(graph, ImportedNode, "Imported node"); + addLegendNode(graph, ImportedBuiltinFilePathNode, "Imported builtin module"); + addLegendNode(graph, ImportedFilePathNode, "Imported module"); + addLegendNode(graph, ImportsFilePathNode, "Module importing selected node"); + addLegendNode(graph, FilePathCenterNode, "Selected node"); + + return graph; +} + +void Diagram::addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text) +{ + util::Graph::Node node = graph_.createNode(); + graph_.setNodeAttribute(node, "label", ""); + decorateNode(graph_, node, nodeType); + + util::Graph::Node explanation = graph_.createNode(); + graph_.setNodeAttribute(explanation, "shape", "none"); + + util::Graph::Edge edge = graph_.createEdge(node, explanation); + graph_.setEdgeAttribute(edge, "style", "invis"); + + graph_.setNodeAttribute(explanation, "label", text); +} } // python } // language } // service diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 7ad66e761..db21fa1e9 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -28,6 +28,9 @@ namespace python util::Graph getFunctionCallDiagram(const model::PYName& pyname); util::Graph getModuleDiagram(const core::FileId& fileId); util::Graph getUsageDiagram(const model::PYName& pyname); + util::Graph getFunctionCallDiagramLegend(); + util::Graph getUsageDiagramLegend(); + util::Graph getModuleDiagramLegend(); enum NodeType { CenterNode, @@ -47,6 +50,7 @@ namespace python core::ProjectServiceHandler m_projectService; std::map m_subgraphs; void addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType); + void addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text); void decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeType& nodeType); util::Graph::Node addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph); util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType); diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 506a46e93..fc18081ad 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -167,6 +167,25 @@ void PythonServiceHandler::getDiagramLegend( const std::int32_t diagramId_) { LOG(info) << "[PYSERVICE] " << __func__; + python::Diagram diagram(_db, _datadir, _context); + + util::Graph graph = [&]() + { + switch (diagramId_) + { + case FUNCTION_CALL: + return diagram.getFunctionCallDiagramLegend(); + case FUNCTION_USAGE: + case CLASS_USAGE: + return diagram.getUsageDiagramLegend(); + default: + return util::Graph(); + } + }(); + + if (graph.nodeCount() != 0) + return_ = graph.output(util::Graph::SVG); + return; } @@ -209,6 +228,22 @@ void PythonServiceHandler::getFileDiagramLegend( const std::int32_t diagramId_) { LOG(info) << "[PYSERVICE] " << __func__; + python::Diagram diagram(_db, _datadir, _context); + + util::Graph graph = [&]() + { + switch (diagramId_) + { + case MODULE_DEPENDENCY: + return diagram.getModuleDiagramLegend(); + default: + return util::Graph(); + } + }(); + + if (graph.nodeCount() != 0) + return_ = graph.output(util::Graph::SVG); + return; } From 1220c4e1f945c8b48f57a8384a7b5e01f3ee8e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 6 Oct 2024 13:41:25 +0200 Subject: [PATCH 108/156] [Diagrams] Remove border from certain file path nodes --- plugins/python/service/src/diagram.cpp | 16 ++++++++++------ plugins/python/service/src/diagram.h | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index c58a576cd..90866b362 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -311,8 +311,8 @@ util::Graph Diagram::getFunctionCallDiagramLegend() util::Graph graph; graph.setAttribute("rankdir", "LR"); - addLegendNode(graph, ImportedBuiltinFilePathNode, "Builtin file path"); - addLegendNode(graph, ImportedFilePathNode, "File path"); + addLegendNode(graph, ImportedBuiltinFilePathNode, "Builtin file path", false); + addLegendNode(graph, ImportedFilePathNode, "File path", false); addLegendNode(graph, FunctionCallNode, "Function call without definition"); addLegendNode(graph, FunctionCallDefinitionNode, "Function call"); addLegendNode(graph, FunctionCallerNode, "Line of code calling selected node"); @@ -327,8 +327,8 @@ util::Graph Diagram::getUsageDiagramLegend() util::Graph graph; graph.setAttribute("rankdir", "LR"); - addLegendNode(graph, ImportedBuiltinFilePathNode, "Builtin module path"); - addLegendNode(graph, ImportedFilePathNode, "Module path"); + addLegendNode(graph, ImportedBuiltinFilePathNode, "Builtin module path", false); + addLegendNode(graph, ImportedFilePathNode, "Module path", false); addLegendNode(graph, FunctionCallNode, "Function call"); addLegendNode(graph, UsageNode, "Usage node"); addLegendNode(graph, CenterNode, "Selected node"); @@ -341,7 +341,7 @@ util::Graph Diagram::getModuleDiagramLegend() util::Graph graph; graph.setAttribute("rankdir", "LR"); - addLegendNode(graph, ImportedNode, "Imported node"); + addLegendNode(graph, ImportedNode, "Imported definition"); addLegendNode(graph, ImportedBuiltinFilePathNode, "Imported builtin module"); addLegendNode(graph, ImportedFilePathNode, "Imported module"); addLegendNode(graph, ImportsFilePathNode, "Module importing selected node"); @@ -350,12 +350,16 @@ util::Graph Diagram::getModuleDiagramLegend() return graph; } -void Diagram::addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text) +void Diagram::addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text, bool shape) { util::Graph::Node node = graph_.createNode(); graph_.setNodeAttribute(node, "label", ""); decorateNode(graph_, node, nodeType); + if (!shape) { + graph_.setNodeAttribute(node, "shape", "plaintext"); + } + util::Graph::Node explanation = graph_.createNode(); graph_.setNodeAttribute(explanation, "shape", "none"); diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index db21fa1e9..f122cae7d 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -50,7 +50,7 @@ namespace python core::ProjectServiceHandler m_projectService; std::map m_subgraphs; void addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType); - void addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text); + void addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text, bool shape = true); void decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeType& nodeType); util::Graph::Node addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph); util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType); From ede556b6dab467c5e33ee3d44ea1cefaabb424a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 6 Oct 2024 17:36:19 +0200 Subject: [PATCH 109/156] [Diagrams] Class diagram --- .../service/include/service/pythonservice.h | 3 +- plugins/python/service/src/diagram.cpp | 101 +++++++++++++++--- plugins/python/service/src/diagram.h | 3 + plugins/python/service/src/pythonservice.cpp | 6 ++ 4 files changed, 98 insertions(+), 15 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index c9ba4a197..f22046dc0 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -219,7 +219,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf FUNCTION_CALL, MODULE_DEPENDENCY, FUNCTION_USAGE, - CLASS_USAGE + CLASS_USAGE, + CLASS_OVERVIEW }; model::PYName queryNodeByID(const std::string& id); diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 90866b362..334ae48bc 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -176,17 +176,14 @@ void Diagram::addFunctionNode(util::Graph& graph_, const util::Graph::Node& cent } } -util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph) +util::Graph::Subgraph Diagram::getFileSubgraph(util::Graph& graph_, const model::PYName& pyname) { - const util::Graph::Subgraph subgraph = [&]() - { - if(!addSubgraph) return util::Graph::Subgraph(); - const core::FileId fileId = std::to_string(pyname.file_id); auto it = m_subgraphs.find(fileId); - if (it != m_subgraphs.end()) + if (it != m_subgraphs.end()) { return it->second; + } core::FileInfo fileInfo; try { @@ -195,22 +192,24 @@ util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYNam return util::Graph::Subgraph(); } - util::Graph::Subgraph subgraph - = graph_.getOrCreateSubgraph("cluster_" + fileInfo.path); + util::Graph::Subgraph subgraph = graph_.getOrCreateSubgraph("cluster_" + fileInfo.path); + graph_.setSubgraphAttribute(subgraph, "id", fileInfo.id); const std::string pathColor = (pyname.is_builtin && pyname.is_definition) ? "dodgerblue" : "limegreen"; - - graph_.setSubgraphAttribute(subgraph, "id", fileInfo.id); const std::string coloredLabel = "
" + fileInfo.path + "
"; graph_.setSubgraphAttribute(subgraph, "label", coloredLabel, true); - m_subgraphs.insert(it, std::make_pair(fileInfo.path, subgraph)); + m_subgraphs.emplace(fileInfo.path, subgraph); return subgraph; - }(); +} + +util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph) +{ + const util::Graph::Subgraph subgraph = (addSubgraph) ? getFileSubgraph(graph_, pyname) : util::Graph::Subgraph(); util::Graph::Node node = graph_.getOrCreateNode(std::to_string(pyname.id), subgraph); @@ -360,14 +359,88 @@ void Diagram::addLegendNode(util::Graph& graph_, const NodeType& nodeType, const graph_.setNodeAttribute(node, "shape", "plaintext"); } - util::Graph::Node explanation = graph_.createNode(); + const util::Graph::Node explanation = graph_.createNode(); graph_.setNodeAttribute(explanation, "shape", "none"); - util::Graph::Edge edge = graph_.createEdge(node, explanation); + const util::Graph::Edge edge = graph_.createEdge(node, explanation); graph_.setEdgeAttribute(edge, "style", "invis"); graph_.setNodeAttribute(explanation, "label", text); } + +util::Graph Diagram::getClassDiagram(const model::PYName& pyname) +{ + util::Graph graph; + graph.setAttribute("rankdir", "BT"); + + const util::Graph::Subgraph subgraph = getFileSubgraph(graph, pyname); + util::Graph::Node classNode = graph.getOrCreateNode(std::to_string(pyname.id), subgraph); + graph.setNodeAttribute(classNode, "shape", "plaintext"); + + graph.setNodeAttribute(classNode, "label", getClassTable(pyname), true); + + return graph; +} + +std::string Diagram::getClassTable(const model::PYName& pyname) +{ + auto getVisibility = [](const std::string& str) + { + if (str.substr(0, 6) == "def __" && str.substr(0, 12) != "def __init__") { + return "- "; + } else { + return "+ "; + } + }; + + auto getSignature = [this](const model::PYName& p) + { + const size_t opening = p.value.find('('); + if (p.value.substr(0, 3) != "def" || opening == std::string::npos) { + return p.value; + } + + std::string sign = p.value.substr(0, opening + 1); + + // Query params + const std::vector params = m_pythonService.queryReferences(std::to_string(p.id), PythonServiceHandler::PARAMETER); + + // Add params to signature + bool first = true; + for (const model::PYName& e : params) { + if (first) { + first = false; + } else { + sign += " "; + } + + sign += e.value; + } + + sign += ')'; + return sign; + }; + + // Query nodes + const std::vector data_members = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::DATA_MEMBER); + const std::vector methods = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::METHOD); + + std::string label = ""; + + label += ""; + + label += ""; + label += "
" + pyname.value + "
"; + for (const model::PYName& p : data_members) { + label += getVisibility(p.value) + getSignature(p) + "
"; + } + label += "
"; + for (const model::PYName& p : methods) { + label += getVisibility(p.value) + getSignature(p) + "
"; + } + label += "
"; + return label; +} } // python } // language } // service diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index f122cae7d..3aa00b3e9 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -28,6 +28,7 @@ namespace python util::Graph getFunctionCallDiagram(const model::PYName& pyname); util::Graph getModuleDiagram(const core::FileId& fileId); util::Graph getUsageDiagram(const model::PYName& pyname); + util::Graph getClassDiagram(const model::PYName& pyname); util::Graph getFunctionCallDiagramLegend(); util::Graph getUsageDiagramLegend(); util::Graph getModuleDiagramLegend(); @@ -55,6 +56,8 @@ namespace python util::Graph::Node addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph); util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType); util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo); + util::Graph::Subgraph getFileSubgraph(util::Graph& graph_, const model::PYName& pyname); + std::string getClassTable(const model::PYName& pyname); }; } // python } // language diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index fc18081ad..24b4857a1 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -119,6 +119,10 @@ void PythonServiceHandler::getDiagramTypes( return_.emplace("Function call", FUNCTION_CALL); } + if (pyname.is_definition == true && pyname.type == "class") { + return_.emplace("Class Overview", CLASS_OVERVIEW); + } + // Usage diagrams const size_t count = PythonServiceHandler::queryReferences(astNodeId_, USAGE).size(); @@ -151,6 +155,8 @@ void PythonServiceHandler::getDiagram( case FUNCTION_USAGE: case CLASS_USAGE: return diagram.getUsageDiagram(pyname); + case CLASS_OVERVIEW: + return diagram.getClassDiagram(pyname); default: return util::Graph(); } From 3ca692ee9af97a44e6ecda9a91fe903dea6e9fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 6 Oct 2024 18:42:47 +0200 Subject: [PATCH 110/156] [PyParser] astparam --- plugins/python/parser/pyparser/asthelper.py | 16 ++++++++++++++++ plugins/python/parser/pyparser/pyname.py | 2 ++ plugins/python/service/src/pythonservice.cpp | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index 89f05d71d..5b7d2c1ce 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -6,12 +6,14 @@ class ASTHelper: astNodes: List[ast.AST] calls: List[ast.Call] imports: List[ast.Import | ast.ImportFrom] + functions: List[ast.FunctionDef] source: str def __init__(self, source): self.astNodes = [] self.calls = [] self.imports = [] + self.functions = [] self.source = source try: @@ -19,12 +21,16 @@ def __init__(self, source): self.astNodes = list(ast.walk(tree)) self.calls = self.__getFunctionCalls() self.imports = self.__getImports() + self.functions = self.__getFunctions() except: pass def __getFunctionCalls(self) -> List[ast.Call]: return cast(List[ast.Call], list(filter(lambda e : isinstance(e, ast.Call), self.astNodes))) + def __getFunctions(self) -> List[ast.FunctionDef]: + return cast(List[ast.FunctionDef], list(filter(lambda e : isinstance(e, ast.FunctionDef), self.astNodes))) + def __getImports(self) -> List[ast.Import | ast.ImportFrom]: return cast(List[ast.Import | ast.ImportFrom], list(filter(lambda e : isinstance(e, ast.Import) or isinstance(e, ast.ImportFrom), self.astNodes))) @@ -63,3 +69,13 @@ def isImport(self, pos: PosInfo): return False + def isFunctionParam(self, pos: PosInfo): + for func in self.functions: + if isinstance(func.args, ast.arguments) and func.args.args: + for e in func.args.args: + if (e.lineno == pos.line_start and + e.end_lineno == pos.line_end and + e.col_offset == pos.column_start): + return True + + return False diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index ea42b4b22..faf8eb4e0 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -111,6 +111,8 @@ def getNodeInfo(self): if self.asthelper: node["is_call"] = self.asthelper.isFunctionCall(self.pos) node["is_import"] = self.asthelper.isImport(self.pos) + if node["type"] == "param": + node["type"] = "astparam" if self.asthelper.isFunctionParam(self.pos) else "param" node["parent_function"] = self.__getParentFunction() diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 24b4857a1..75fab9653 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -471,7 +471,7 @@ std::vector PythonServiceHandler::queryReferences(const core::Ast nodes = _db->query((odb::query::id == pyname.parent_function) + order_by); break; case PARAMETER: - nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "param" && odb::query::is_definition == true) + order_by); + nodes = _db->query((odb::query::parent == pyname.ref_id && odb::query::type == "astparam" && odb::query::is_definition == true) + order_by); break; case CALLER: nodes = _db->query((odb::query::ref_id == pyname.ref_id && odb::query::is_definition == false && odb::query::is_call == true && odb::query::id != pyname.id) + order_by); From ecefcd76fec38b2b5e551c6dfdd5d85f557fcfc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 9 Oct 2024 11:29:29 +0200 Subject: [PATCH 111/156] [PyParser] PYName path, move getHashName to parserutil --- plugins/python/parser/pyparser/parserutil.py | 10 ++++++- plugins/python/parser/pyparser/pyname.py | 28 +++++++++----------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/plugins/python/parser/pyparser/parserutil.py b/plugins/python/parser/pyparser/parserutil.py index f564bef6f..15707e64a 100644 --- a/plugins/python/parser/pyparser/parserutil.py +++ b/plugins/python/parser/pyparser/parserutil.py @@ -1,4 +1,12 @@ -def fnvHash(str): +from hashlib import sha1 +from posinfo import PosInfo + +def getHashName(path, pos: PosInfo) -> int: + s = f"{path}|{pos.line_start}|{pos.line_end}|{pos.column_start}|{pos.column_end}".encode("utf-8") + hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff + return hash + +def fnvHash(str) -> int: hash = 14695981039346656037 for c in str: diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index faf8eb4e0..5b4d6c0da 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -1,7 +1,6 @@ from jedi.api.classes import Name from typing import List -from hashlib import sha1 -from parserutil import fnvHash +from parserutil import fnvHash, getHashName from parserlog import log, bcolors from asthelper import ASTHelper from posinfo import PosInfo @@ -10,6 +9,7 @@ class PYName: id: int + path: str | None name: Name pos: PosInfo refid: int @@ -19,8 +19,9 @@ class PYName: def __init__(self, name: Name): self.name = name + self.path = str(self.name.module_path) if self.name.module_path else None self.pos = self.__getNamePosInfo() - self.hashName = self.__getHashName() + self.hashName = getHashName(self.path, self.pos) self.refid = self.hashName self.defs = [] self.asthelper = None @@ -53,8 +54,8 @@ def addConfig(self, config: ParserConfig): return self def appendModulePath(self, import_list): - if self.name.module_path: - import_list.append(str(self.name.module_path)) + if self.path: + import_list.append(self.path) return self @@ -75,10 +76,10 @@ def __getNamePosInfo(self) -> PosInfo: else: pos.value = self.name.get_line_code()[pos.column_start:] - if (self.name.module_path and + if (self.path and pos.line_start == 0 and pos.line_end == 0 and pos.column_start == 0 and pos.column_end == 0): - pos.value = str(self.name.module_path).split("/")[-1] + pos.value = self.path.split("/")[-1] return pos @@ -87,7 +88,6 @@ def getNodeInfo(self): node["id"] = self.hashName node["ref_id"] = self.refid node["module_name"] = self.name.module_name - node["module_path"] = str(self.name.module_path) node["full_name"] = self.name.full_name if self.name.full_name else "" node["line_start"] = self.pos.line_start @@ -118,17 +118,15 @@ def getNodeInfo(self): return node - def __getHashName(self) -> int: - s = f"{str(self.name.module_path)}|{self.pos.line_start}|{self.pos.line_end}|{self.pos.column_start}|{self.pos.column_end}".encode("utf-8") - hash = int(sha1(s).hexdigest(), 16) & 0xffffffffffffffff - return hash - def __getFileId(self): - return fnvHash(str(self.name.module_path)) + if self.path: + return fnvHash(self.path) + else: + return 0 def __reportMissingDefinition(self, result): if not self.name.is_definition() and self.name.type == 'module': - log(f"{bcolors.FAIL}Missing {self.name.description} (file = {str(self.name.module_path)} line = {self.pos.line_start})") + log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.path} line = {self.pos.line_start})") result["status"] = "partial" def __getParentFunction(self): From 8dac26c9f582ec3fce0a40de01d137a0c6e8c187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 9 Oct 2024 14:07:42 +0200 Subject: [PATCH 112/156] [PyParser] NodeInfo, ASThelper path --- plugins/python/parser/pyparser/asthelper.py | 4 +- plugins/python/parser/pyparser/nodeinfo.py | 26 +++++++++ plugins/python/parser/pyparser/parser.py | 9 ++-- .../python/parser/pyparser/parserconfig.py | 2 +- plugins/python/parser/pyparser/pyname.py | 53 +++++++++---------- plugins/python/parser/src/pythonparser.cpp | 34 ++++++------ 6 files changed, 77 insertions(+), 51 deletions(-) create mode 100644 plugins/python/parser/pyparser/nodeinfo.py diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index 5b7d2c1ce..c14de2f33 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -7,13 +7,15 @@ class ASTHelper: calls: List[ast.Call] imports: List[ast.Import | ast.ImportFrom] functions: List[ast.FunctionDef] + path: str source: str - def __init__(self, source): + def __init__(self, path, source): self.astNodes = [] self.calls = [] self.imports = [] self.functions = [] + self.path = path self.source = source try: diff --git a/plugins/python/parser/pyparser/nodeinfo.py b/plugins/python/parser/pyparser/nodeinfo.py new file mode 100644 index 000000000..d97b6365d --- /dev/null +++ b/plugins/python/parser/pyparser/nodeinfo.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass + +@dataclass +class NodeInfo: + id: int = 0 + ref_id: int = 0 + file_id: int = 0 + + module_name: str = "" + full_name: str = "" + + parent: int = 0 + parent_function: int = 0 + + line_start: int = 1 + line_end: int = 1 + column_start: int = 1 + column_end: int = 1 + value: str = "" + + type: str = "" + type_hint: str = "" + is_definition: bool = False + is_builtin: bool = False + is_call: bool = False + is_import: bool = False diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 796cab473..53d5a8614 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -7,6 +7,7 @@ from asthelper import ASTHelper from pyname import PYName from parserconfig import ParserConfig +from nodeinfo import NodeInfo from pyreference import PYReference from pybuiltin import PYBuiltin @@ -75,7 +76,7 @@ def parse(path: str, config: ParserConfig): PYBuiltin.findBuiltins(config) - nodes: dict[int, PYName] = {} + nodes: dict[int, NodeInfo] = {} with open(path) as f: try: @@ -84,7 +85,7 @@ def parse(path: str, config: ParserConfig): script = jedi.Script(source, path=path, project=config.project) names = script.get_names(references = True, all_scopes = True) - asthelper = ASTHelper(source) + asthelper = ASTHelper(path, source) pyref = PYReference(config, script, names) for x in names: @@ -109,6 +110,6 @@ def parse(path: str, config: ParserConfig): return result -def putInMap(hashmap, node): - hashmap[node["id"]] = node +def putInMap(hashmap: dict[int, NodeInfo], node: NodeInfo): + hashmap[node.id] = node diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py index 94835a403..d2434613a 100644 --- a/plugins/python/parser/pyparser/parserconfig.py +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -7,7 +7,7 @@ class ParserConfig: venv_path: str | None = None project: Project | None = None debug: bool = True - stack_trace: bool = False + stack_trace: bool = True safe_env: bool = False type_hint: bool = False file_refs: bool = True diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 5b4d6c0da..f10f87ca6 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -6,6 +6,7 @@ from posinfo import PosInfo from pybuiltin import PYBuiltin from parserconfig import ParserConfig +from nodeinfo import NodeInfo class PYName: id: int @@ -83,38 +84,34 @@ def __getNamePosInfo(self) -> PosInfo: return pos - def getNodeInfo(self): - node = {} - node["id"] = self.hashName - node["ref_id"] = self.refid - node["module_name"] = self.name.module_name - node["full_name"] = self.name.full_name if self.name.full_name else "" - - node["line_start"] = self.pos.line_start - node["line_end"] = self.pos.line_end - node["column_start"] = self.pos.column_start + 1 - node["column_end"] = self.pos.column_end + 1 - node["value"] = self.pos.value - - node["type"] = self.name.type - node["is_definition"] = self.name.is_definition() - node["file_id"] = self.__getFileId() - node["type_hint"] = self.__getNameTypeHint() - node["is_builtin"] = PYBuiltin.isBuiltin(self.name) or any(list(map(lambda x : PYBuiltin.isBuiltin(x.name), self.defs))) + def getNodeInfo(self) -> NodeInfo: + node = NodeInfo() + node.id = self.hashName + node.ref_id = self.refid + node.module_name = self.name.module_name + node.full_name = self.name.full_name if self.name.full_name else "" + + node.line_start = self.pos.line_start + node.line_end = self.pos.line_end + node.column_start = self.pos.column_start + 1 + node.column_end = self.pos.column_end + 1 + node.value = self.pos.value + + node.type = self.name.type + node.is_definition = self.name.is_definition() + node.file_id = self.__getFileId() + node.type_hint = self.__getNameTypeHint() + node.is_builtin = PYBuiltin.isBuiltin(self.name) or any(list(map(lambda x : PYBuiltin.isBuiltin(x.name), self.defs))) parent = self.name.parent() - node["parent"] = PYName(parent).hashName if parent else node["id"] - - node["is_call"] = False - node["is_import"] = False + node.parent = PYName(parent).hashName if parent else node.id + node.parent_function = self.__getParentFunction() if self.asthelper: - node["is_call"] = self.asthelper.isFunctionCall(self.pos) - node["is_import"] = self.asthelper.isImport(self.pos) - if node["type"] == "param": - node["type"] = "astparam" if self.asthelper.isFunctionParam(self.pos) else "param" - - node["parent_function"] = self.__getParentFunction() + node.is_call = self.asthelper.isFunctionCall(self.pos) + node.is_import = self.asthelper.isImport(self.pos) + if node.type == "param": + node.type = "astparam" if self.asthelper.isFunctionParam(self.pos) else "param" return node diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index eb4023565..5cde5cf42 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -109,23 +109,23 @@ void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseR python::object node = nodes[i]; model::PYName pyname; - pyname.id = python::extract(node["id"]); - pyname.ref_id = python::extract(node["ref_id"]); - pyname.parent = python::extract(node["parent"]); - pyname.parent_function = python::extract(node["parent_function"]); - pyname.full_name = python::extract(node["full_name"]); - pyname.is_definition = python::extract(node["is_definition"]); - pyname.is_import = python::extract(node["is_import"]); - pyname.is_builtin = python::extract(node["is_builtin"]); - pyname.line_start = python::extract(node["line_start"]); - pyname.line_end = python::extract(node["line_end"]); - pyname.column_start = python::extract(node["column_start"]); - pyname.column_end = python::extract(node["column_end"]); - pyname.file_id = python::extract(node["file_id"]); - pyname.value = python::extract(node["value"]); - pyname.type = python::extract(node["type"]); - pyname.type_hint = python::extract(node["type_hint"]); - pyname.is_call = python::extract(node["is_call"]); + pyname.id = python::extract(node.attr("id")); + pyname.ref_id = python::extract(node.attr("ref_id")); + pyname.parent = python::extract(node.attr("parent")); + pyname.parent_function = python::extract(node.attr("parent_function")); + pyname.full_name = python::extract(node.attr("full_name")); + pyname.is_definition = python::extract(node.attr("is_definition")); + pyname.is_import = python::extract(node.attr("is_import")); + pyname.is_builtin = python::extract(node.attr("is_builtin")); + pyname.line_start = python::extract(node.attr("line_start")); + pyname.line_end = python::extract(node.attr("line_end")); + pyname.column_start = python::extract(node.attr("column_start")); + pyname.column_end = python::extract(node.attr("column_end")); + pyname.file_id = python::extract(node.attr("file_id")); + pyname.value = python::extract(node.attr("value")); + pyname.type = python::extract(node.attr("type")); + pyname.type_hint = python::extract(node.attr("type_hint")); + pyname.is_call = python::extract(node.attr("is_call")); // Put in map if(map.find(pyname.id) == map.end()) From a6b12dd2a7022d62ea5ecdf7332a95bf3ea49f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 12 Oct 2024 22:33:41 +0200 Subject: [PATCH 113/156] [PyParser] Annotations --- plugins/python/parser/pyparser/asthelper.py | 54 +++++++++++++++++++++ plugins/python/parser/pyparser/parser.py | 3 ++ 2 files changed, 57 insertions(+) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index c14de2f33..bc257e10d 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -1,6 +1,8 @@ import ast from typing import cast, List from posinfo import PosInfo +from nodeinfo import NodeInfo +from parserutil import fnvHash, getHashName class ASTHelper: astNodes: List[ast.AST] @@ -9,6 +11,7 @@ class ASTHelper: functions: List[ast.FunctionDef] path: str source: str + lines: List[str] def __init__(self, path, source): self.astNodes = [] @@ -16,7 +19,9 @@ def __init__(self, path, source): self.imports = [] self.functions = [] self.path = path + self.file_id = fnvHash(self.path) self.source = source + self.lines = source.split("\n") try: tree = ast.parse(source) @@ -81,3 +86,52 @@ def isFunctionParam(self, pos: PosInfo): return True return False + + def getAnnotations(self): + results = [] + + for func in self.functions: + if not (isinstance(func.returns, ast.Subscript) and + func.lineno and + func.end_lineno and + func.col_offset and + func.end_col_offset): + continue + + sub = func.returns + line_start = sub.lineno - 1 + line_end = sub.end_lineno - 1 if sub.end_lineno else line_start + col_start = sub.col_offset + col_end = sub.end_col_offset if sub.end_col_offset else col_start + + if line_start == line_end: + return_type = self.lines[line_start][col_start:col_end] + else: + return_type = self.lines[line_start][col_start:] + for l in range(line_start + 1, line_end): + return_type += self.lines[l] + return_type += self.lines[line_end][:col_end] + + if return_type: + funcpos = PosInfo(line_start=func.lineno, line_end=func.end_lineno, column_start=func.col_offset, column_end=func.end_col_offset) + subpos = PosInfo(line_start=sub.lineno, line_end=line_end + 1, column_start=sub.col_offset, column_end=col_end) + + funchash = getHashName(self.path, funcpos) + subhash = getHashName(self.path, subpos) + + nodeinfo = NodeInfo() + nodeinfo.id = subhash + nodeinfo.ref_id = subhash + nodeinfo.parent = funchash + nodeinfo.parent_function = funchash + nodeinfo.line_start = subpos.line_start + nodeinfo.line_end = subpos.line_end + nodeinfo.column_start = subpos.column_start + nodeinfo.column_end = subpos.column_end + nodeinfo.file_id = self.file_id + nodeinfo.type = "annotation" + nodeinfo.value = return_type + + results.append(nodeinfo) + + return results diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 53d5a8614..eb3fa074c 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -88,6 +88,9 @@ def parse(path: str, config: ParserConfig): asthelper = ASTHelper(path, source) pyref = PYReference(config, script, names) + for e in asthelper.getAnnotations(): + putInMap(nodes, e) + for x in names: defs = pyref.getDefs(x) refs = pyref.getFileRefs(x) From 4d726dc40741f6b4c9899a833233eca2ce372f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 12 Oct 2024 22:35:20 +0200 Subject: [PATCH 114/156] Graphviz Cairo SVG rendering --- util/include/util/graph.h | 2 +- util/src/graph.cpp | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/util/include/util/graph.h b/util/include/util/graph.h index c58ae5914..0285014c2 100644 --- a/util/include/util/graph.h +++ b/util/include/util/graph.h @@ -27,7 +27,7 @@ struct GraphPimpl; class Graph { public: - enum Format {DOT, SVG}; + enum Format {DOT, SVG, CAIRO_SVG}; typedef std::string Node; typedef std::string Edge; diff --git a/util/src/graph.cpp b/util/src/graph.cpp index 7d15c5936..19f315623 100644 --- a/util/src/graph.cpp +++ b/util/src/graph.cpp @@ -278,10 +278,23 @@ std::string Graph::output(Graph::Format format_) const gvLayout(_graphPimpl->_gvc, _graphPimpl->_graph, "dot"); + const char* render_format; + switch (format_) { + case Graph::DOT: + render_format = "dot"; + break; + case Graph::SVG: + render_format = "svg"; + break; + case Graph::CAIRO_SVG: + render_format = "svg:cairo"; + break; + } + gvRenderData( _graphPimpl->_gvc, _graphPimpl->_graph, - format_ == Graph::DOT ? "dot" : "svg", + render_format, result, length); From 45bf725ca644afc776aa9a1da53591bcb8f0ab7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 12 Oct 2024 22:36:23 +0200 Subject: [PATCH 115/156] [Diagrams] Class diagram return annotation, syntax highlight --- .../service/include/service/pythonservice.h | 2 + plugins/python/service/src/diagram.cpp | 60 +++++++++++++++---- plugins/python/service/src/pythonservice.cpp | 12 +++- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index f22046dc0..d6fd9f264 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -200,6 +200,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf PARENT, PARENT_FUNCTION, + + ANNOTATION, }; enum FileReferenceType diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 334ae48bc..dcd2d0eea 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -375,6 +375,7 @@ util::Graph Diagram::getClassDiagram(const model::PYName& pyname) const util::Graph::Subgraph subgraph = getFileSubgraph(graph, pyname); util::Graph::Node classNode = graph.getOrCreateNode(std::to_string(pyname.id), subgraph); + graph.setNodeAttribute(classNode, "fontname", "Noto Serif"); graph.setNodeAttribute(classNode, "shape", "plaintext"); graph.setNodeAttribute(classNode, "label", getClassTable(pyname), true); @@ -386,21 +387,48 @@ std::string Diagram::getClassTable(const model::PYName& pyname) { auto getVisibility = [](const std::string& str) { - if (str.substr(0, 6) == "def __" && str.substr(0, 12) != "def __init__") { - return "- "; + if (str.substr(0, 6) == "def __") { + return std::string("- "); } else { - return "+ "; + return std::string("+ "); } }; - auto getSignature = [this](const model::PYName& p) + auto highlightVariable = [](const std::string& str, const std::string& baseColor) + { + std::string p = (str.back() == ',') ? str.substr(0, str.size() - 1) : str; + + const size_t col = p.find(":"); + const size_t eq = p.find("="); + + if (col == std::string::npos) { + p = "" + p + ""; + } else if (eq == std::string::npos) { + p = "" + p.substr(0, col) + "" + ":" + + "" + p.substr(col + 1) + ""; + } else { + p = "" + p.substr(0, col) + "" + ":" + + "" + p.substr(col + 1, eq - col - 1) + "" + + "" + p.substr(eq) + ""; + } + + if (str.back() == ',') { + p += ','; + } + + return p; + }; + + auto getSignature = [this, &highlightVariable](const model::PYName& p) { const size_t opening = p.value.find('('); if (p.value.substr(0, 3) != "def" || opening == std::string::npos) { - return p.value; + return highlightVariable(p.value, "black"); } - std::string sign = p.value.substr(0, opening + 1); + // Remove "def" + std::string sign = p.value.substr(3, opening - 3); + sign += '('; // Query params const std::vector params = m_pythonService.queryReferences(std::to_string(p.id), PythonServiceHandler::PARAMETER); @@ -408,16 +436,28 @@ std::string Diagram::getClassTable(const model::PYName& pyname) // Add params to signature bool first = true; for (const model::PYName& e : params) { + // Skip param "self" + if (first && e.value.substr(0,4) == "self") { + continue; + } + if (first) { first = false; } else { sign += " "; } - sign += e.value; + sign += highlightVariable(e.value, "darkgreen"); } sign += ')'; + + // Query return annotation + const std::vector annotations = m_pythonService.queryReferences(std::to_string(p.id), PythonServiceHandler::ANNOTATION); + if(annotations.size() == 1) { + sign += " -> " + annotations[0].value + ""; + } + return sign; }; @@ -425,17 +465,17 @@ std::string Diagram::getClassTable(const model::PYName& pyname) const std::vector data_members = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::DATA_MEMBER); const std::vector methods = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::METHOD); - std::string label = ""; + std::string label = "
" + pyname.value + "
"; label += ""; label += ""; label += "
" + pyname.value + "
"; for (const model::PYName& p : data_members) { - label += getVisibility(p.value) + getSignature(p) + "
"; + label += "
" + getVisibility(p.value) + getSignature(p) + "
"; } label += "
"; for (const model::PYName& p : methods) { - label += getVisibility(p.value) + getSignature(p) + "
"; + label += "
" + getVisibility(p.value) + getSignature(p) + "
"; } label += "
"; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 75fab9653..35d2ea73e 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -162,8 +162,13 @@ void PythonServiceHandler::getDiagram( } }(); - if (graph.nodeCount() != 0) - return_ = graph.output(util::Graph::SVG); + if (graph.nodeCount() != 0) { + if (diagramId_ == CLASS_OVERVIEW) { + return_ = graph.output(util::Graph::CAIRO_SVG); + } else { + return_ = graph.output(util::Graph::SVG); + } + } return; } @@ -479,6 +484,9 @@ std::vector PythonServiceHandler::queryReferences(const core::Ast case THIS_CALLS: nodes = _db->query((odb::query::parent == pyname.id && odb::query::is_call == true) + order_by); break; + case ANNOTATION: + nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "annotation") + order_by); + break; } return std::vector(nodes.begin(), nodes.end()); From 97de0b7aab3c7974b30af700088e4e67a8cd6c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 13 Oct 2024 18:29:33 +0200 Subject: [PATCH 116/156] [PyParser] getSubclass --- plugins/python/parser/pyparser/asthelper.py | 38 ++++++++++++++++++--- plugins/python/parser/pyparser/pyname.py | 6 ++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index bc257e10d..cbbfc3d04 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -9,6 +9,7 @@ class ASTHelper: calls: List[ast.Call] imports: List[ast.Import | ast.ImportFrom] functions: List[ast.FunctionDef] + classes: List[ast.ClassDef] path: str source: str lines: List[str] @@ -18,6 +19,7 @@ def __init__(self, path, source): self.calls = [] self.imports = [] self.functions = [] + self.classes = [] self.path = path self.file_id = fnvHash(self.path) self.source = source @@ -26,9 +28,11 @@ def __init__(self, path, source): try: tree = ast.parse(source) self.astNodes = list(ast.walk(tree)) + self.calls = self.__getFunctionCalls() self.imports = self.__getImports() self.functions = self.__getFunctions() + self.classes = self.__getClasses() except: pass @@ -38,6 +42,9 @@ def __getFunctionCalls(self) -> List[ast.Call]: def __getFunctions(self) -> List[ast.FunctionDef]: return cast(List[ast.FunctionDef], list(filter(lambda e : isinstance(e, ast.FunctionDef), self.astNodes))) + def __getClasses(self) -> List[ast.ClassDef]: + return cast(List[ast.ClassDef], list(filter(lambda e : isinstance(e, ast.ClassDef), self.astNodes))) + def __getImports(self) -> List[ast.Import | ast.ImportFrom]: return cast(List[ast.Import | ast.ImportFrom], list(filter(lambda e : isinstance(e, ast.Import) or isinstance(e, ast.ImportFrom), self.astNodes))) @@ -53,7 +60,7 @@ def isFunctionCall(self, pos: PosInfo): return True elif (isinstance(func, ast.Attribute)): - if func.end_col_offset: + if isinstance(func.end_col_offset, int): col_start = func.end_col_offset - len(func.attr) else: col_start = 0 @@ -87,15 +94,36 @@ def isFunctionParam(self, pos: PosInfo): return False + def getSubclass(self, pos: PosInfo) -> int | None: + for cls in self.classes: + if not (isinstance(cls.lineno, int) and + isinstance(cls.end_lineno, int) and + isinstance(cls.col_offset, int)): + continue + + for e in cls.bases: + if (isinstance(e, ast.Name) and + e.lineno == pos.line_start and + e.end_lineno == pos.line_end and + e.col_offset == pos.column_start and + e.end_col_offset == pos.column_end): + + col_end = len(self.lines[cls.end_lineno - 1]) + subpos = PosInfo(line_start=cls.lineno, line_end=cls.end_lineno, column_start=cls.col_offset, column_end=col_end) + subhash = getHashName(self.path, subpos) + return subhash + + return None + def getAnnotations(self): results = [] for func in self.functions: if not (isinstance(func.returns, ast.Subscript) and - func.lineno and - func.end_lineno and - func.col_offset and - func.end_col_offset): + isinstance(func.lineno, int) and + isinstance(func.end_lineno, int) and + isinstance(func.col_offset, int) and + isinstance(func.end_col_offset, int)): continue sub = func.returns diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index f10f87ca6..5cd7f0054 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -110,9 +110,15 @@ def getNodeInfo(self) -> NodeInfo: if self.asthelper: node.is_call = self.asthelper.isFunctionCall(self.pos) node.is_import = self.asthelper.isImport(self.pos) + if node.type == "param": node.type = "astparam" if self.asthelper.isFunctionParam(self.pos) else "param" + subclass = self.asthelper.getSubclass(self.pos) + if subclass: + node.type = "baseclass" + node.parent = subclass + return node def __getFileId(self): From b4dd3bdd0643b585390a5c6a448123f0538c5eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 13 Oct 2024 18:30:04 +0200 Subject: [PATCH 117/156] [PythonService] Base class --- .../python/service/include/service/pythonservice.h | 2 ++ plugins/python/service/src/pythonservice.cpp | 7 +++++++ plugins/python/webgui/css/pythonplugin.css | 14 +++++++++----- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index d6fd9f264..ebb0f82f2 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -202,6 +202,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf PARENT_FUNCTION, ANNOTATION, + + BASE_CLASS, }; enum FileReferenceType diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 35d2ea73e..525cb6afc 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -287,6 +287,7 @@ void PythonServiceHandler::getReferenceTypes( { return_.emplace("Method", METHOD); return_.emplace("Data member", DATA_MEMBER); + return_.emplace("Base class", BASE_CLASS); } } @@ -487,6 +488,12 @@ std::vector PythonServiceHandler::queryReferences(const core::Ast case ANNOTATION: nodes = _db->query((odb::query::parent == pyname.id && odb::query::type == "annotation") + order_by); break; + case BASE_CLASS: + odb::result bases = _db->query((odb::query::parent == pyname.id && odb::query::type == "baseclass")); + const std::vector bases_refs = PythonServiceHandler::transformReferences(std::vector(bases.begin(), bases.end()), model::REF_ID); + + nodes = _db->query((odb::query::ref_id.in_range(bases_refs.begin(), bases_refs.end()) && odb::query::is_definition == true && odb::query::is_import == false) + order_by); + break; } return std::vector(nodes.begin(), nodes.end()); diff --git a/plugins/python/webgui/css/pythonplugin.css b/plugins/python/webgui/css/pythonplugin.css index c705ad51c..4349563fc 100644 --- a/plugins/python/webgui/css/pythonplugin.css +++ b/plugins/python/webgui/css/pythonplugin.css @@ -1,19 +1,23 @@ -.icon-Builtin::before{ +.icon-Builtin::before { content : '\e017' } -.icon-Type-hint::before{ +.icon-Type-hint::before { content : '\e040' } -.icon-Full-name::before{ +.icon-Full-name::before { content: '\e001'; } -.icon-Parent::before{ +.icon-Parent::before { + content : '\e034' +} + +.icon-Base-class::before { content : '\e033' } -.icon-Function-call::before{ +.icon-Function-call::before { content : '\e028' } From 262d0325ff57a43b7f0878cd931fe28f3f123003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 13 Oct 2024 19:11:30 +0200 Subject: [PATCH 118/156] [Diagrams] Class diagram base class --- plugins/python/service/src/diagram.cpp | 34 ++++++++++++++++++-------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index dcd2d0eea..607ced7c4 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -373,12 +373,29 @@ util::Graph Diagram::getClassDiagram(const model::PYName& pyname) util::Graph graph; graph.setAttribute("rankdir", "BT"); - const util::Graph::Subgraph subgraph = getFileSubgraph(graph, pyname); - util::Graph::Node classNode = graph.getOrCreateNode(std::to_string(pyname.id), subgraph); - graph.setNodeAttribute(classNode, "fontname", "Noto Serif"); - graph.setNodeAttribute(classNode, "shape", "plaintext"); + auto setAttributes = [this, &graph](const util::Graph::Node& node, const model::PYName& p) { + graph.setNodeAttribute(node, "fontname", "Noto Serif"); + graph.setNodeAttribute(node, "shape", "plaintext"); + graph.setNodeAttribute(node, "label", getClassTable(p), true); + }; + + util::Graph::Node classNode = graph.getOrCreateNode(std::to_string(pyname.id), getFileSubgraph(graph, pyname)); + setAttributes(classNode, pyname); + + // Query baseclasses + const std::vector bases = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::BASE_CLASS); + + for (const model::PYName& p : bases) { + if (p.type != "class") { + continue; + } - graph.setNodeAttribute(classNode, "label", getClassTable(pyname), true); + util::Graph::Node node = graph.getOrCreateNode(std::to_string(p.id), getFileSubgraph(graph, p)); + setAttributes(node, p); + + util::Graph::Edge edge = graph.createEdge(classNode, node); + graph.setEdgeAttribute(edge, "arrowhead", "empty"); + } return graph; } @@ -396,6 +413,7 @@ std::string Diagram::getClassTable(const model::PYName& pyname) auto highlightVariable = [](const std::string& str, const std::string& baseColor) { + // Remove comma std::string p = (str.back() == ',') ? str.substr(0, str.size() - 1) : str; const size_t col = p.find(":"); @@ -412,10 +430,6 @@ std::string Diagram::getClassTable(const model::PYName& pyname) "" + p.substr(eq) + ""; } - if (str.back() == ',') { - p += ','; - } - return p; }; @@ -444,7 +458,7 @@ std::string Diagram::getClassTable(const model::PYName& pyname) if (first) { first = false; } else { - sign += " "; + sign += ", "; } sign += highlightVariable(e.value, "darkgreen"); From d757ce35af3d1b8742a6ff08b54015bb1b3bf33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 15 Oct 2024 12:53:42 +0200 Subject: [PATCH 119/156] [PyParser] Rewrite definition handling --- plugins/python/parser/pyparser/parser.py | 12 ++++++++++-- plugins/python/parser/pyparser/pyname.py | 6 ------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index eb3fa074c..be9333d29 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -77,6 +77,7 @@ def parse(path: str, config: ParserConfig): PYBuiltin.findBuiltins(config) nodes: dict[int, NodeInfo] = {} + imports: dict[str, bool] = {} with open(path) as f: try: @@ -98,8 +99,14 @@ def parse(path: str, config: ParserConfig): putInMap(nodes, PYName(x).addConfig(config).addDefs(defs, result).addRefs(refs).addASTHelper(asthelper).getNodeInfo()) for d in defs: - if not (d.hashName in nodes): - putInMap(nodes, d.addConfig(config).appendModulePath(result["imports"]).getNodeInfo()) + if not (d.path): + continue + + # Builtin or library definition + if ((config.venv_path and d.path.startswith(config.venv_path)) or + not (d.path.startswith(config.root_path))): + putInMap(nodes, d.addConfig(config).getNodeInfo()) + imports[d.path] = True except: log(f"{bcolors.FAIL}Failed to parse file: {path}") @@ -107,6 +114,7 @@ def parse(path: str, config: ParserConfig): traceback.print_exc() result["nodes"] = list(nodes.values()) + result["imports"] = list(imports.keys()) if len(result["nodes"]) == 0: result["status"] = "none" diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 5cd7f0054..68b58746b 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -54,12 +54,6 @@ def addConfig(self, config: ParserConfig): self.config = config return self - def appendModulePath(self, import_list): - if self.path: - import_list.append(self.path) - - return self - def __getNamePosInfo(self) -> PosInfo: pos = PosInfo() From 7bbb69e2fc822ada53d03a5bd25ac3b7c58be45d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 15 Oct 2024 13:43:51 +0200 Subject: [PATCH 120/156] [PyParser] More config options --- plugins/python/parser/pyparser/asthelper.py | 10 +++++++++- plugins/python/parser/pyparser/parser.py | 2 +- plugins/python/parser/pyparser/parserconfig.py | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index cbbfc3d04..f5b70aac2 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -3,6 +3,7 @@ from posinfo import PosInfo from nodeinfo import NodeInfo from parserutil import fnvHash, getHashName +from parserconfig import ParserConfig class ASTHelper: astNodes: List[ast.AST] @@ -14,7 +15,7 @@ class ASTHelper: source: str lines: List[str] - def __init__(self, path, source): + def __init__(self, path: str, source: str, config: ParserConfig): self.astNodes = [] self.calls = [] self.imports = [] @@ -24,6 +25,7 @@ def __init__(self, path, source): self.file_id = fnvHash(self.path) self.source = source self.lines = source.split("\n") + self.config = config try: tree = ast.parse(source) @@ -95,6 +97,9 @@ def isFunctionParam(self, pos: PosInfo): return False def getSubclass(self, pos: PosInfo) -> int | None: + if not (self.config.ast_inheritance): + return None + for cls in self.classes: if not (isinstance(cls.lineno, int) and isinstance(cls.end_lineno, int) and @@ -116,6 +121,9 @@ def getSubclass(self, pos: PosInfo) -> int | None: return None def getAnnotations(self): + if not (self.config.ast_annotations): + return [] + results = [] for func in self.functions: diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index be9333d29..62036d92f 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -86,7 +86,7 @@ def parse(path: str, config: ParserConfig): script = jedi.Script(source, path=path, project=config.project) names = script.get_names(references = True, all_scopes = True) - asthelper = ASTHelper(path, source) + asthelper = ASTHelper(path, source, config) pyref = PYReference(config, script, names) for e in asthelper.getAnnotations(): diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py index d2434613a..b5db39709 100644 --- a/plugins/python/parser/pyparser/parserconfig.py +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -11,3 +11,5 @@ class ParserConfig: safe_env: bool = False type_hint: bool = False file_refs: bool = True + ast_annotations: bool = True + ast_inheritance: bool = True From 856587e0ad2dcae27724d040b3501b33a6f80425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 15 Oct 2024 16:55:42 +0200 Subject: [PATCH 121/156] [PyParser] AST function signature --- plugins/python/parser/pyparser/asthelper.py | 112 ++++++++++++++---- .../python/parser/pyparser/parserconfig.py | 3 +- plugins/python/parser/pyparser/pyname.py | 4 + 3 files changed, 98 insertions(+), 21 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index f5b70aac2..32b490a5a 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -120,6 +120,44 @@ def getSubclass(self, pos: PosInfo) -> int | None: return None + def __getASTValue(self, node: ast.Subscript | ast.Attribute) -> PosInfo | None: + line_start = node.lineno - 1 + line_end = node.end_lineno - 1 if node.end_lineno else line_start + col_start = node.col_offset + col_end = node.end_col_offset if node.end_col_offset else col_start + + if line_start == line_end: + value = self.lines[line_start][col_start:col_end] + else: + value = self.lines[line_start][col_start:] + for l in range(line_start + 1, line_end): + value += self.lines[l] + value += self.lines[line_end][:col_end] + + if value: + return PosInfo(line_start=line_start + 1, line_end=line_end + 1, column_start=node.col_offset, column_end=col_end, value=value) + else: + return None + + def __getFunctionReturnAnnotation(self, func: ast.FunctionDef) -> PosInfo | None: + posinfo = PosInfo() + if func.returns: + posinfo.line_start = func.returns.lineno + posinfo.line_end = func.returns.end_lineno if func.returns.end_lineno else func.returns.lineno + posinfo.column_start = func.returns.col_offset + posinfo.column_end = func.returns.end_col_offset if func.returns.end_col_offset else func.returns.col_offset + + if isinstance(func.returns, ast.Subscript) or isinstance(func.returns, ast.Attribute): + return self.__getASTValue(func.returns) + elif isinstance(func.returns, ast.Name) and func.returns.id: + posinfo.value = func.returns.id + return posinfo + elif isinstance(func.returns, ast.Constant): + posinfo.value = str(func.returns.value) + return posinfo + + return None + def getAnnotations(self): if not (self.config.ast_annotations): return [] @@ -127,31 +165,16 @@ def getAnnotations(self): results = [] for func in self.functions: - if not (isinstance(func.returns, ast.Subscript) and - isinstance(func.lineno, int) and + if not (isinstance(func.lineno, int) and isinstance(func.end_lineno, int) and isinstance(func.col_offset, int) and isinstance(func.end_col_offset, int)): continue - sub = func.returns - line_start = sub.lineno - 1 - line_end = sub.end_lineno - 1 if sub.end_lineno else line_start - col_start = sub.col_offset - col_end = sub.end_col_offset if sub.end_col_offset else col_start - - if line_start == line_end: - return_type = self.lines[line_start][col_start:col_end] - else: - return_type = self.lines[line_start][col_start:] - for l in range(line_start + 1, line_end): - return_type += self.lines[l] - return_type += self.lines[line_end][:col_end] - - if return_type: - funcpos = PosInfo(line_start=func.lineno, line_end=func.end_lineno, column_start=func.col_offset, column_end=func.end_col_offset) - subpos = PosInfo(line_start=sub.lineno, line_end=line_end + 1, column_start=sub.col_offset, column_end=col_end) + subpos = self.__getFunctionReturnAnnotation(func) + if subpos: + funcpos = PosInfo(line_start=func.lineno, line_end=func.end_lineno, column_start=func.col_offset, column_end=func.end_col_offset) funchash = getHashName(self.path, funcpos) subhash = getHashName(self.path, subpos) @@ -166,8 +189,57 @@ def getAnnotations(self): nodeinfo.column_end = subpos.column_end nodeinfo.file_id = self.file_id nodeinfo.type = "annotation" - nodeinfo.value = return_type + nodeinfo.value = subpos.value results.append(nodeinfo) return results + + def getFunctionSignature(self, pos: PosInfo) -> str | None: + if not (self.config.ast_function_signature): + return None + + for func in self.functions: + if not (isinstance(func.lineno, int) and + isinstance(func.end_lineno, int) and + isinstance(func.col_offset, int) and + isinstance(func.end_col_offset, int)): + continue + + if not (func.lineno == pos.line_start and + func.end_lineno == pos.line_end and + func.col_offset == pos.column_start and + func.end_col_offset == pos.column_end): + continue + + sign = "def " + func.name + sign += "(" + + first = True + for arg in func.args.args: + if first: + first = False + else: + sign += ", " + + sign += arg.arg + + param_annotation: str | None = None + if isinstance(arg.annotation, ast.Name): + param_annotation = arg.annotation.id + elif isinstance(arg.annotation, ast.Subscript) or isinstance(arg.annotation, ast.Attribute): + sub = self.__getASTValue(arg.annotation) + param_annotation = sub.value if sub else None + + if param_annotation: + sign += ": " + param_annotation + + sign += ")" + + return_annotation = self.__getFunctionReturnAnnotation(func) + if return_annotation: + sign += " -> " + return_annotation.value + + return sign + + return None diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py index b5db39709..8ce0cb2f8 100644 --- a/plugins/python/parser/pyparser/parserconfig.py +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -6,10 +6,11 @@ class ParserConfig: root_path: str venv_path: str | None = None project: Project | None = None - debug: bool = True + debug: bool = False stack_trace: bool = True safe_env: bool = False type_hint: bool = False file_refs: bool = True ast_annotations: bool = True ast_inheritance: bool = True + ast_function_signature: bool = True diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 68b58746b..08ace11d7 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -108,6 +108,10 @@ def getNodeInfo(self) -> NodeInfo: if node.type == "param": node.type = "astparam" if self.asthelper.isFunctionParam(self.pos) else "param" + if node.type == "function" and node.is_definition and not (node.is_import or node.is_builtin): + sign = self.asthelper.getFunctionSignature(self.pos) + node.value = sign if sign else node.value + subclass = self.asthelper.getSubclass(self.pos) if subclass: node.type = "baseclass" From e339e33f9ba81baa0bf82abc76503b1c5df4781b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 15 Oct 2024 18:57:04 +0200 Subject: [PATCH 122/156] [PyParser] Added more config options --- plugins/python/parser/pyparser/asthelper.py | 15 ++++++++++----- plugins/python/parser/pyparser/parserconfig.py | 3 +++ plugins/python/parser/pyparser/pyname.py | 4 +++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index 32b490a5a..cf3b86220 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -30,13 +30,18 @@ def __init__(self, path: str, source: str, config: ParserConfig): try: tree = ast.parse(source) self.astNodes = list(ast.walk(tree)) + except: + pass + + if config.ast: + if self.config.ast_function_call: + self.calls = self.__getFunctionCalls() + + if self.config.ast_import: + self.imports = self.__getImports() - self.calls = self.__getFunctionCalls() - self.imports = self.__getImports() self.functions = self.__getFunctions() self.classes = self.__getClasses() - except: - pass def __getFunctionCalls(self) -> List[ast.Call]: return cast(List[ast.Call], list(filter(lambda e : isinstance(e, ast.Call), self.astNodes))) @@ -159,7 +164,7 @@ def __getFunctionReturnAnnotation(self, func: ast.FunctionDef) -> PosInfo | None return None def getAnnotations(self): - if not (self.config.ast_annotations): + if not (self.config.ast and self.config.ast_annotations): return [] results = [] diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py index 8ce0cb2f8..9488f0a9a 100644 --- a/plugins/python/parser/pyparser/parserconfig.py +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -11,6 +11,9 @@ class ParserConfig: safe_env: bool = False type_hint: bool = False file_refs: bool = True + ast: bool = True + ast_function_call: bool = True + ast_import: bool = True ast_annotations: bool = True ast_inheritance: bool = True ast_function_signature: bool = True diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 08ace11d7..832cfa3bb 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -47,7 +47,9 @@ def addRefs(self, refs: List['PYName']): return self def addASTHelper(self, asthelper: ASTHelper): - self.asthelper = asthelper + if self.config and self.config.ast: + self.asthelper = asthelper + return self def addConfig(self, config: ParserConfig): From b926226d159cc49ed69f8118b60c74956337f873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 15 Oct 2024 19:14:58 +0200 Subject: [PATCH 123/156] [PyParser] log_config --- plugins/python/parser/pyparser/parser.py | 19 ++---------- plugins/python/parser/pyparser/parserlog.py | 33 ++++++++++++++++++++- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 62036d92f..5ad9cd920 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -3,7 +3,7 @@ import multiprocessing import traceback from itertools import repeat -from parserlog import log, bcolors +from parserlog import log, bcolors, log_config from asthelper import ASTHelper from pyname import PYName from parserconfig import ParserConfig @@ -12,22 +12,9 @@ from pybuiltin import PYBuiltin def parseProject(root_path, venv_path, sys_path, n_proc): - log(f"Parsing project: {root_path}") config = ParserConfig(root_path=root_path) - - if config.debug: - log(f"{bcolors.WARNING}Parsing in debug mode!") - - if not config.safe_env: - log(f"{bcolors.WARNING}Creating Python environment in unsafe mode!") - - if config.type_hint: - log(f"{bcolors.OKGREEN}Type hint support enabled!") - else: - log(f"{bcolors.OKBLUE}Type hint support disabled!") - - if config.stack_trace: - log(f"{bcolors.WARNING}Stack trace enabled!") + log(f"Parsing project: {root_path}") + log_config(config) try: if venv_path: diff --git a/plugins/python/parser/pyparser/parserlog.py b/plugins/python/parser/pyparser/parserlog.py index 54ecdbaa2..5563a4d7a 100644 --- a/plugins/python/parser/pyparser/parserlog.py +++ b/plugins/python/parser/pyparser/parserlog.py @@ -1,4 +1,5 @@ import datetime +from parserconfig import ParserConfig def log(msg): date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -13,4 +14,34 @@ class bcolors: FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' - UNDERLINE = '\033[4m' \ No newline at end of file + UNDERLINE = '\033[4m' + +def log_config(config: ParserConfig): + if config.debug: + log(f"{bcolors.WARNING}Parsing in debug mode!") + + if config.type_hint: + log(f"{bcolors.OKGREEN}Type hint support enabled!") + else: + log(f"{bcolors.OKBLUE}Type hint support disabled!") + + if config.stack_trace: + log(f"{bcolors.WARNING}Stack trace enabled!") + + if not (config.ast): + log(f"{bcolors.WARNING}All AST modules disabled!") + + if not (config.ast_function_call): + log(f"{bcolors.WARNING}AST function call parsing disabled!") + + if not (config.ast_import): + log(f"{bcolors.WARNING}AST import parsing disabled!") + + if not (config.ast_inheritance): + log(f"{bcolors.WARNING}AST inheritance parsing disabled!") + + if not (config.ast_annotations): + log(f"{bcolors.WARNING}AST annotation parsing disabled!") + + if not (config.ast_function_signature): + log(f"{bcolors.WARNING}AST function signature parsing disabled!") From e9c950113e767569fa31e404e4ad7ba6d68e311f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 15 Oct 2024 19:37:16 +0200 Subject: [PATCH 124/156] [PyParser] ParseResult --- .../parser/include/pythonparser/pythonparser.h | 4 ++-- plugins/python/parser/pyparser/parser.py | 16 ++++++---------- plugins/python/parser/pyparser/parseresult.py | 8 ++++++++ plugins/python/parser/pyparser/pyname.py | 5 +++-- plugins/python/parser/src/pythonparser.cpp | 12 ++++++------ 5 files changed, 25 insertions(+), 20 deletions(-) create mode 100644 plugins/python/parser/pyparser/parseresult.py diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index 06ba100f8..f0c7fda7f 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -26,14 +26,14 @@ class PythonParser : public AbstractParser virtual ~PythonParser(); virtual bool parse() override; private: - struct ParseResult { + struct ParseResultStats { std::uint32_t partial; std::uint32_t full; }; python::object m_py_module; bool accept(const std::string& path_); - void processFile(const python::object& obj, PYNameMap& map, ParseResult& parse_result); + void processFile(const python::object& obj, PYNameMap& map, ParseResultStats& parse_result); void parseProject(const std::string& root_path); }; diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 5ad9cd920..f39134772 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -8,6 +8,7 @@ from pyname import PYName from parserconfig import ParserConfig from nodeinfo import NodeInfo +from parseresult import ParseResult from pyreference import PYReference from pybuiltin import PYBuiltin @@ -54,12 +55,7 @@ def parseProject(root_path, venv_path, sys_path, n_proc): return results def parse(path: str, config: ParserConfig): - result = { - "path": path, - "status": "full", - "nodes": [], - "imports": [] - } + result: ParseResult = ParseResult(path=path) PYBuiltin.findBuiltins(config) @@ -100,11 +96,11 @@ def parse(path: str, config: ParserConfig): if config.stack_trace: traceback.print_exc() - result["nodes"] = list(nodes.values()) - result["imports"] = list(imports.keys()) + result.nodes = list(nodes.values()) + result.imports = list(imports.keys()) - if len(result["nodes"]) == 0: - result["status"] = "none" + if len(result.nodes) == 0: + result.status = "none" return result diff --git a/plugins/python/parser/pyparser/parseresult.py b/plugins/python/parser/pyparser/parseresult.py new file mode 100644 index 000000000..cb5fbece6 --- /dev/null +++ b/plugins/python/parser/pyparser/parseresult.py @@ -0,0 +1,8 @@ +from dataclasses import dataclass + +@dataclass +class ParseResult: + path: str + status: str = "full" + nodes = [] + imports = [] diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 832cfa3bb..75d2e4fea 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -7,6 +7,7 @@ from pybuiltin import PYBuiltin from parserconfig import ParserConfig from nodeinfo import NodeInfo +from parseresult import ParseResult class PYName: id: int @@ -127,10 +128,10 @@ def __getFileId(self): else: return 0 - def __reportMissingDefinition(self, result): + def __reportMissingDefinition(self, result: ParseResult): if not self.name.is_definition() and self.name.type == 'module': log(f"{bcolors.FAIL}Missing {self.name.description} (file = {self.path} line = {self.pos.line_start})") - result["status"] = "partial" + result.status = "partial" def __getParentFunction(self): try: diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 5cde5cf42..06f704f27 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -39,7 +39,7 @@ PythonParser::PythonParser(ParserContext& ctx_): AbstractParser(ctx_) void PythonParser::parseProject(const std::string& root_path) { PYNameMap map; - ParseResult parse_result; + ParseResultStats parse_result; parse_result.full = 0; parse_result.partial = 0; @@ -96,12 +96,12 @@ bool PythonParser::accept(const std::string& path_) return ext == ".py"; } -void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseResult& parse_result) +void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseResultStats& parse_result) { try { - python::object nodes = obj["nodes"]; - const std::string status = python::extract(obj["status"]); - const std::string path = python::extract(obj["path"]); + python::object nodes = obj.attr("nodes"); + const std::string status = python::extract(obj.attr("status")); + const std::string path = python::extract(obj.attr("path")); const int len = python::len(nodes); for (int i = 0; i < len; i++) @@ -154,7 +154,7 @@ void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseR // Additional paths (example: builtin definition paths) // These files need to be added to db - python::object imports = obj["imports"]; + python::object imports = obj.attr("imports"); for (int i = 0; i < python::len(imports); i++) { std::string p = python::extract(imports[i]); From c0d02b31d864c7121c5b52df36d3ef13934e08eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 16 Oct 2024 14:59:06 +0200 Subject: [PATCH 125/156] [PyParser] __getPosValue, getFunctionParam --- plugins/python/parser/pyparser/asthelper.py | 40 +++++++++++++-------- plugins/python/parser/pyparser/pyname.py | 5 ++- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index cf3b86220..59576a353 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -90,16 +90,15 @@ def isImport(self, pos: PosInfo): return False - def isFunctionParam(self, pos: PosInfo): + def getFunctionParam(self, pos: PosInfo) -> str | None: for func in self.functions: if isinstance(func.args, ast.arguments) and func.args.args: for e in func.args.args: if (e.lineno == pos.line_start and - e.end_lineno == pos.line_end and e.col_offset == pos.column_start): - return True + return self.__getPosValue(pos) - return False + return None def getSubclass(self, pos: PosInfo) -> int | None: if not (self.config.ast_inheritance): @@ -125,22 +124,33 @@ def getSubclass(self, pos: PosInfo) -> int | None: return None - def __getASTValue(self, node: ast.Subscript | ast.Attribute) -> PosInfo | None: - line_start = node.lineno - 1 - line_end = node.end_lineno - 1 if node.end_lineno else line_start - col_start = node.col_offset - col_end = node.end_col_offset if node.end_col_offset else col_start + def __getPosValue(self, pos: PosInfo) -> str | None: + if not (pos.line_start >= 1 and pos.line_end >= 1): + return None - if line_start == line_end: - value = self.lines[line_start][col_start:col_end] + if pos.line_start == pos.line_end: + value = self.lines[pos.line_start - 1][pos.column_start:pos.column_end] else: - value = self.lines[line_start][col_start:] - for l in range(line_start + 1, line_end): + value = self.lines[pos.line_start - 1][pos.column_start:] + for l in range(pos.line_start, pos.line_end - 1): value += self.lines[l] - value += self.lines[line_end][:col_end] + value += self.lines[pos.line_end - 1][:pos.column_end] + + if value: + return value + else: + return None + + def __getASTValue(self, node: ast.Subscript | ast.Attribute) -> PosInfo | None: + line_end = node.end_lineno if node.end_lineno else node.lineno + col_end = node.end_col_offset if node.end_col_offset else node.col_offset + + pos = PosInfo(line_start=node.lineno, line_end=line_end, column_start=node.col_offset, column_end=col_end) + value = self.__getPosValue(pos) if value: - return PosInfo(line_start=line_start + 1, line_end=line_end + 1, column_start=node.col_offset, column_end=col_end, value=value) + pos.value = value + return pos else: return None diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 75d2e4fea..3d197bf1a 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -109,7 +109,10 @@ def getNodeInfo(self) -> NodeInfo: node.is_import = self.asthelper.isImport(self.pos) if node.type == "param": - node.type = "astparam" if self.asthelper.isFunctionParam(self.pos) else "param" + param = self.asthelper.getFunctionParam(self.pos) + if param: + node.type = "astparam" + node.value = param if node.type == "function" and node.is_definition and not (node.is_import or node.is_builtin): sign = self.asthelper.getFunctionSignature(self.pos) From cb7d2138b362f9282d7aaa72dd1e2ff837f6fd07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 16 Oct 2024 16:23:41 +0200 Subject: [PATCH 126/156] [Diagrams] Class diagram variable highlight fix --- plugins/python/service/src/diagram.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 607ced7c4..e598a7ee1 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -418,16 +418,17 @@ std::string Diagram::getClassTable(const model::PYName& pyname) const size_t col = p.find(":"); const size_t eq = p.find("="); + const size_t hashtag = p.find("#"); - if (col == std::string::npos) { - p = "" + p + ""; - } else if (eq == std::string::npos) { - p = "" + p.substr(0, col) + "" + ":" + - "" + p.substr(col + 1) + ""; - } else { + if (col != std::string::npos && eq != std::string::npos && col < eq && hashtag == std::string::npos) { p = "" + p.substr(0, col) + "" + ":" + "" + p.substr(col + 1, eq - col - 1) + "" + "" + p.substr(eq) + ""; + } else if (col != std::string::npos && eq == std::string::npos && hashtag == std::string::npos) { + p = "" + p.substr(0, col) + "" + ":" + + "" + p.substr(col + 1) + ""; + } else { + p = "" + p + ""; } return p; From 0c51573e90175128e5ffda0536e70421bd20103c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 16 Oct 2024 17:15:20 +0200 Subject: [PATCH 127/156] [PyParser] Expand __getASTValue --- plugins/python/parser/pyparser/asthelper.py | 44 +++++++++------------ 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index 59576a353..71c5d5790 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -141,12 +141,19 @@ def __getPosValue(self, pos: PosInfo) -> str | None: else: return None - def __getASTValue(self, node: ast.Subscript | ast.Attribute) -> PosInfo | None: + def __getASTValue(self, node: ast.expr) -> PosInfo | None: line_end = node.end_lineno if node.end_lineno else node.lineno col_end = node.end_col_offset if node.end_col_offset else node.col_offset pos = PosInfo(line_start=node.lineno, line_end=line_end, column_start=node.col_offset, column_end=col_end) - value = self.__getPosValue(pos) + value = None + + if isinstance(node, ast.Subscript) or isinstance(node, ast.Attribute): + value = self.__getPosValue(pos) + elif isinstance(node, ast.Name): + value = node.id + elif isinstance(node, ast.Constant): + value = str(node.value) if value: pos.value = value @@ -155,23 +162,16 @@ def __getASTValue(self, node: ast.Subscript | ast.Attribute) -> PosInfo | None: return None def __getFunctionReturnAnnotation(self, func: ast.FunctionDef) -> PosInfo | None: - posinfo = PosInfo() if func.returns: - posinfo.line_start = func.returns.lineno - posinfo.line_end = func.returns.end_lineno if func.returns.end_lineno else func.returns.lineno - posinfo.column_start = func.returns.col_offset - posinfo.column_end = func.returns.end_col_offset if func.returns.end_col_offset else func.returns.col_offset - - if isinstance(func.returns, ast.Subscript) or isinstance(func.returns, ast.Attribute): return self.__getASTValue(func.returns) - elif isinstance(func.returns, ast.Name) and func.returns.id: - posinfo.value = func.returns.id - return posinfo - elif isinstance(func.returns, ast.Constant): - posinfo.value = str(func.returns.value) - return posinfo + else: + return None - return None + def __getArgumentAnnotation(self, arg: ast.arg) -> PosInfo | None: + if arg.annotation: + return self.__getASTValue(arg.annotation) + else: + return None def getAnnotations(self): if not (self.config.ast and self.config.ast_annotations): @@ -239,19 +239,13 @@ def getFunctionSignature(self, pos: PosInfo) -> str | None: sign += arg.arg - param_annotation: str | None = None - if isinstance(arg.annotation, ast.Name): - param_annotation = arg.annotation.id - elif isinstance(arg.annotation, ast.Subscript) or isinstance(arg.annotation, ast.Attribute): - sub = self.__getASTValue(arg.annotation) - param_annotation = sub.value if sub else None - + param_annotation: PosInfo | None = self.__getArgumentAnnotation(arg) if param_annotation: - sign += ": " + param_annotation + sign += ": " + param_annotation.value sign += ")" - return_annotation = self.__getFunctionReturnAnnotation(func) + return_annotation: PosInfo | None = self.__getFunctionReturnAnnotation(func) if return_annotation: sign += " -> " + return_annotation.value From 7cb12e9e34823689dca90ead0fe60c0c21aec3d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 21 Oct 2024 16:28:29 +0200 Subject: [PATCH 128/156] [PythonParser] CLI arguments --- .../include/pythonparser/pythonparser.h | 1 + plugins/python/parser/pyparser/parser.py | 39 ++++++++++------ .../python/parser/pyparser/parserconfig.py | 4 +- plugins/python/parser/src/pythonparser.cpp | 44 ++++++++++++++----- 4 files changed, 64 insertions(+), 24 deletions(-) diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index f0c7fda7f..f9f0e2f8b 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -2,6 +2,7 @@ #define CC_PARSER_PYTHONPARSER_H #include +#include #include #include #include diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index f39134772..1ef2517ed 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -12,26 +12,39 @@ from pyreference import PYReference from pybuiltin import PYBuiltin -def parseProject(root_path, venv_path, sys_path, n_proc): - config = ParserConfig(root_path=root_path) - log(f"Parsing project: {root_path}") +def parseProject(settings, n_proc): + config = ParserConfig( + root_path=settings["root_path"], + venv_path=settings["venv_path"], + sys_path=settings["sys_path"], + debug=settings["debug"], + stack_trace=settings["stack_trace"], + type_hint=settings["type_hint"], + ast=settings["ast"], + ast_function_call=settings["ast_function_call"], + ast_import=settings["ast_import"], + ast_annotations=settings["ast_annotations"], + ast_inheritance=settings["ast_inheritance"], + ast_function_signature=settings["ast_function_signature"] + ) + + log(f"Parsing project: {config.root_path}") log_config(config) try: - if venv_path: - jedi.create_environment(venv_path, safe = config.safe_env) - config.venv_path = venv_path - log(f"{bcolors.OKGREEN}Using virtual environment: {venv_path}") + if config.venv_path: + jedi.create_environment(config.venv_path, safe = config.safe_env) + log(f"{bcolors.OKGREEN}Using virtual environment: {config.venv_path}") else: - venv_path = None + config.venv_path = None - if sys_path: - log(f"{bcolors.OKGREEN}Using additional syspath: {sys_path}") + if config.sys_path: + log(f"{bcolors.OKGREEN}Using additional syspath: {config.sys_path}") - config.project = jedi.Project(path = root_path, environment_path = venv_path, added_sys_path = sys_path) + config.project = jedi.Project(path = config.root_path, environment_path = config.venv_path, added_sys_path = config.sys_path) except: - log(f"{bcolors.FAIL}Failed to use virtual environment: {venv_path}") + log(f"{bcolors.FAIL}Failed to use virtual environment: {config.venv_path}") if config.stack_trace: traceback.print_exc() @@ -39,7 +52,7 @@ def parseProject(root_path, venv_path, sys_path, n_proc): log(f"{bcolors.OKGREEN}Using {n_proc} process to parse project") - for root, dirs, files in os.walk(root_path): + for root, dirs, files in os.walk(config.root_path): for file in files: p = os.path.join(root, file) ext = os.path.splitext(p)[1] diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py index 9488f0a9a..57211285e 100644 --- a/plugins/python/parser/pyparser/parserconfig.py +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -1,13 +1,15 @@ from dataclasses import dataclass +from typing import List from jedi import Project @dataclass class ParserConfig: root_path: str + sys_path: List[str] venv_path: str | None = None project: Project | None = None debug: bool = False - stack_trace: bool = True + stack_trace: bool = False safe_env: bool = False type_hint: bool = False file_refs: bool = True diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 06f704f27..b1f7ae27e 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -1,9 +1,6 @@ #include #include #include -#include -#include -#include namespace cc { @@ -44,7 +41,6 @@ void PythonParser::parseProject(const std::string& root_path) parse_result.partial = 0; try { - std::string venv; python::list sys_path; int n_proc = _ctx.options["jobs"].as(); @@ -57,12 +53,22 @@ void PythonParser::parseProject(const std::string& root_path) } } - if (_ctx.options.count("venvpath")) - { - venv = _ctx.options["venvpath"].as(); - } + python::dict settings; + settings["root_path"] = root_path; + settings["sys_path"] = sys_path; + settings["venv_path"] = (_ctx.options.count("venvpath")) ? _ctx.options["venvpath"].as() : ""; + settings["debug"] = (bool)(_ctx.options.count("debug")); + settings["stack_trace"] = (bool)(_ctx.options.count("stack-trace")); + settings["type_hint"] = (bool)(_ctx.options.count("type-hint")); + settings["ast"] = !(_ctx.options.count("disable-ast")); + settings["ast_function_call"] = !(_ctx.options.count("disable-ast-function-call")); + settings["ast_import"] = !(_ctx.options.count("disable-ast-import")); + settings["ast_annotations"] = !(_ctx.options.count("disable-ast-annotations")); + settings["ast_inheritance"] = !(_ctx.options.count("disable-ast-inheritance")); + settings["ast_function_signature"] = !(_ctx.options.count("disable-ast-function-signature")); + + python::object result_list = m_py_module.attr("parseProject")(settings, n_proc); - python::object result_list = m_py_module.attr("parseProject")(root_path, venv, sys_path, n_proc); for(int i = 0; i < python::len(result_list); i++) { PythonParser::processFile(result_list[i], map, parse_result); @@ -195,7 +201,25 @@ extern "C" ("venvpath", po::value(), "Set 'venvpath' to specify the project's Python virtual environment path.") ("syspath", po::value>(), - "Additional sys path for the parser."); + "Additional sys path for the parser.") + ("stack-trace", + "Enable error stack trace.") + ("type-hint", + "Enable type hint parsing.") + ("disable-ast", + "Disable all AST parsing modules.") + ("disable-ast-function-call", + "Disable AST function call parsing.") + ("disable-ast-import", + "Disable AST import parsing.") + ("disable-ast-annotations", + "Disable AST annotation parsing.") + ("disable-ast-inheritance", + "Disable AST inheritance parsing.") + ("disable-ast-function-signature", + "Disable AST function signature parsing.") + ("debug", + "Enable parsing in debug mode."); return description; } From 67bb581a7c5f140ecc710f72caf5cd503ece1048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 16 Nov 2024 15:30:54 +0100 Subject: [PATCH 129/156] [PyParser] Submodule discovery --- plugins/python/parser/pyparser/parser.py | 36 ++++++++++++------- .../python/parser/pyparser/parserconfig.py | 1 + plugins/python/parser/pyparser/parserlog.py | 3 ++ plugins/python/parser/src/pythonparser.cpp | 3 ++ 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 1ef2517ed..5b4af6fc8 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -20,6 +20,7 @@ def parseProject(settings, n_proc): debug=settings["debug"], stack_trace=settings["stack_trace"], type_hint=settings["type_hint"], + submodule_discovery=settings["submodule_discovery"], ast=settings["ast"], ast_function_call=settings["ast_function_call"], ast_import=settings["ast_import"], @@ -31,6 +32,23 @@ def parseProject(settings, n_proc): log(f"Parsing project: {config.root_path}") log_config(config) + py_files = [] + submodule_map = {} + for root, dirs, files in os.walk(config.root_path): + if config.venv_path and root.startswith(config.venv_path): + continue + + for file in files: + p = os.path.join(root, file) + ext = os.path.splitext(p)[1] + + if ext and ext.lower() == '.py': + py_files.append(p) + + if file == '__init__.py': + parent_dir = os.path.abspath(os.path.join(root, os.pardir)) + submodule_map[parent_dir] = True + try: if config.venv_path: jedi.create_environment(config.venv_path, safe = config.safe_env) @@ -40,6 +58,11 @@ def parseProject(settings, n_proc): if config.sys_path: log(f"{bcolors.OKGREEN}Using additional syspath: {config.sys_path}") + + submodule_sys_path = list(submodule_map.keys()) + if config.submodule_discovery and submodule_sys_path: + log(f"{bcolors.OKBLUE}Submodule discovery results: {submodule_sys_path}") + config.sys_path.extend(submodule_sys_path) config.project = jedi.Project(path = config.root_path, environment_path = config.venv_path, added_sys_path = config.sys_path) @@ -48,21 +71,8 @@ def parseProject(settings, n_proc): if config.stack_trace: traceback.print_exc() - py_files = [] - log(f"{bcolors.OKGREEN}Using {n_proc} process to parse project") - for root, dirs, files in os.walk(config.root_path): - for file in files: - p = os.path.join(root, file) - ext = os.path.splitext(p)[1] - - if ext and ext.lower() == '.py': - py_files.append(p) - - if config.venv_path: - py_files = filter(lambda e : not(e.startswith(config.venv_path)), py_files) - with multiprocessing.Pool(processes=n_proc) as pool: results = pool.starmap(parse, zip(py_files, repeat(config))) return results diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py index 57211285e..e6927c845 100644 --- a/plugins/python/parser/pyparser/parserconfig.py +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -13,6 +13,7 @@ class ParserConfig: safe_env: bool = False type_hint: bool = False file_refs: bool = True + submodule_discovery: bool = True ast: bool = True ast_function_call: bool = True ast_import: bool = True diff --git a/plugins/python/parser/pyparser/parserlog.py b/plugins/python/parser/pyparser/parserlog.py index 5563a4d7a..2bec87be9 100644 --- a/plugins/python/parser/pyparser/parserlog.py +++ b/plugins/python/parser/pyparser/parserlog.py @@ -28,6 +28,9 @@ def log_config(config: ParserConfig): if config.stack_trace: log(f"{bcolors.WARNING}Stack trace enabled!") + if not (config.submodule_discovery): + log(f"{bcolors.WARNING}Submodule discovery disabled!") + if not (config.ast): log(f"{bcolors.WARNING}All AST modules disabled!") diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index b1f7ae27e..180cecb19 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -60,6 +60,7 @@ void PythonParser::parseProject(const std::string& root_path) settings["debug"] = (bool)(_ctx.options.count("debug")); settings["stack_trace"] = (bool)(_ctx.options.count("stack-trace")); settings["type_hint"] = (bool)(_ctx.options.count("type-hint")); + settings["submodule_discovery"] = !(_ctx.options.count("disable-submodule-discovery")); settings["ast"] = !(_ctx.options.count("disable-ast")); settings["ast_function_call"] = !(_ctx.options.count("disable-ast-function-call")); settings["ast_import"] = !(_ctx.options.count("disable-ast-import")); @@ -206,6 +207,8 @@ extern "C" "Enable error stack trace.") ("type-hint", "Enable type hint parsing.") + ("disable-submodule-discovery", + "Disable submodule disovery. Submodules will not be added to sys path.") ("disable-ast", "Disable all AST parsing modules.") ("disable-ast-function-call", From 509b936f0e921723684bc2807da848e6350942c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 17 Nov 2024 15:35:36 +0100 Subject: [PATCH 130/156] [PythonTest] Parser test --- plugins/python/CMakeLists.txt | 3 +- plugins/python/test/CMakeLists.txt | 39 +++++++++++++++++ plugins/python/test/sources/functions.py | 4 ++ plugins/python/test/src/pythonparsertest.cpp | 44 ++++++++++++++++++++ plugins/python/test/src/pythontest.cpp | 20 +++++++++ 5 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 plugins/python/test/CMakeLists.txt create mode 100644 plugins/python/test/sources/functions.py create mode 100644 plugins/python/test/src/pythonparsertest.cpp create mode 100644 plugins/python/test/src/pythontest.cpp diff --git a/plugins/python/CMakeLists.txt b/plugins/python/CMakeLists.txt index 5aa4665f9..ebd6783d8 100644 --- a/plugins/python/CMakeLists.txt +++ b/plugins/python/CMakeLists.txt @@ -1,5 +1,6 @@ add_subdirectory(model) add_subdirectory(parser) add_subdirectory(service) -#add_subdirectory(test) +add_subdirectory(test) + install_webplugin(webgui) diff --git a/plugins/python/test/CMakeLists.txt b/plugins/python/test/CMakeLists.txt new file mode 100644 index 000000000..6384bf211 --- /dev/null +++ b/plugins/python/test/CMakeLists.txt @@ -0,0 +1,39 @@ +if (NOT FUNCTIONAL_TESTING_ENABLED) + fancy_message("Skipping generation of test project pythontest." "yellow" TRUE) + return() +endif() + +fancy_message("Generating test project for pythontest." "blue" TRUE) + +include_directories( + ${PROJECT_SOURCE_DIR}/model/include + ${PROJECT_SOURCE_DIR}/util/include) + +add_executable(pythonparsertest + src/pythontest.cpp + src/pythonparsertest.cpp) + +target_link_libraries(pythonparsertest + ${GTEST_BOTH_LIBRARIES} + pthread + util + model + pythonmodel) + +# Clean up the build folder and the output of the test in a `make clean`. +set_property(DIRECTORY APPEND PROPERTY + ADDITIONAL_MAKE_CLEAN_FILES + "${CMAKE_CURRENT_BINARY_DIR}/workdir") + +# CTest runs the `pythonparsertest` binary with two arguments: +# 1. Shell script which parses the Python project +# 2. Test database connection string +add_test(NAME pythonparser COMMAND pythonparsertest + "echo \"Test database used: ${TEST_DB}\" && \ + ${CMAKE_INSTALL_PREFIX}/bin/CodeCompass_parser \ + --database \"${TEST_DB}\" \ + --name pythonparsertest \ + --input ${CMAKE_CURRENT_SOURCE_DIR}/sources/ \ + --workspace ${CMAKE_CURRENT_BINARY_DIR}/workdir/ \ + --force" + "${TEST_DB}") diff --git a/plugins/python/test/sources/functions.py b/plugins/python/test/sources/functions.py new file mode 100644 index 000000000..df2cf28b6 --- /dev/null +++ b/plugins/python/test/sources/functions.py @@ -0,0 +1,4 @@ +def hello_world(): + print("Hello, world!") + +hello_world() diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp new file mode 100644 index 000000000..1ea37334e --- /dev/null +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -0,0 +1,44 @@ +#include +#include +#include + +#include +#include +#include +#include + +using namespace cc; +extern const char* dbConnectionString; + +class PythonParserTest : public ::testing::Test +{ +public: + PythonParserTest() : + _db(util::connectDatabase(dbConnectionString)), + _transaction(_db) {} + +protected: + std::shared_ptr _db; + util::OdbTransaction _transaction; +}; + +TEST_F(PythonParserTest, FilesAreInDatabase) +{ + _transaction([&, this]() { + model::File file; + + file = _db->query_value(odb::query::filename == "functions.py"); + EXPECT_EQ(file.type, "PY"); + EXPECT_EQ(file.parseStatus, model::File::PSFullyParsed); + }); +} + +TEST_F(PythonParserTest, FunctionDefinitions) +{ + _transaction([&, this]() { + model::PYName pyname; + + pyname = _db->query_value(odb::query::value == "def hello_world()"); + EXPECT_EQ(pyname.is_definition, true); + }); +} diff --git a/plugins/python/test/src/pythontest.cpp b/plugins/python/test/src/pythontest.cpp new file mode 100644 index 000000000..6b1ebbe6a --- /dev/null +++ b/plugins/python/test/src/pythontest.cpp @@ -0,0 +1,20 @@ +#include + +const char* dbConnectionString; + +int main(int argc, char** argv) +{ + if (argc < 3) + { + GTEST_LOG_(FATAL) << "Test arguments missing."; + return 1; + } + + GTEST_LOG_(INFO) << "Testing Python started..."; + system(argv[1]); + + GTEST_LOG_(INFO) << "Using database for tests: " << dbConnectionString; + dbConnectionString = argv[2]; + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 02ac6b9971d7d202ba427fdf94b29dd732fe4091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 17 Nov 2024 20:34:53 +0100 Subject: [PATCH 131/156] [PythonTest] Service test --- plugins/python/service/CMakeLists.txt | 1 + .../service/include/service/pythonservice.h | 1 - plugins/python/service/src/diagram.h | 3 +- plugins/python/service/src/pythonservice.cpp | 1 + plugins/python/test/CMakeLists.txt | 31 ++++++++- plugins/python/test/src/pythonparsertest.cpp | 4 +- plugins/python/test/src/pythonservicetest.cpp | 63 +++++++++++++++++++ 7 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 plugins/python/test/src/pythonservicetest.cpp diff --git a/plugins/python/service/CMakeLists.txt b/plugins/python/service/CMakeLists.txt index 343873a73..17b63f5f2 100644 --- a/plugins/python/service/CMakeLists.txt +++ b/plugins/python/service/CMakeLists.txt @@ -22,6 +22,7 @@ target_link_libraries(pythonservice util model pythonmodel + mongoose projectservice languagethrift ${THRIFT_LIBTHRIFT_LIBRARIES} diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index ebb0f82f2..a0443da89 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -11,7 +11,6 @@ #include #include #include -#include #include diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 3aa00b3e9..d2c1711c9 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -1,9 +1,10 @@ #ifndef CC_SERVICE_PYTHON_DIAGRAM_H #define CC_SERVICE_PYTHON_DIAGRAM_H +#include +#include #include #include -#include #include #include #include diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 525cb6afc..e30ffab25 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include "diagram.h" diff --git a/plugins/python/test/CMakeLists.txt b/plugins/python/test/CMakeLists.txt index 6384bf211..773771795 100644 --- a/plugins/python/test/CMakeLists.txt +++ b/plugins/python/test/CMakeLists.txt @@ -6,13 +6,24 @@ endif() fancy_message("Generating test project for pythontest." "blue" TRUE) include_directories( + ${PLUGIN_DIR}/model/include + ${PLUGIN_DIR}/service/include ${PROJECT_SOURCE_DIR}/model/include - ${PROJECT_SOURCE_DIR}/util/include) + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_BINARY_DIR}/service/language/gen-cpp + ${PROJECT_BINARY_DIR}/service/project/gen-cpp) add_executable(pythonparsertest src/pythontest.cpp src/pythonparsertest.cpp) +add_executable(pythonservicetest + src/pythontest.cpp + src/pythonservicetest.cpp) + +target_compile_options(pythonparsertest PUBLIC -Wno-unknown-pragmas) +target_compile_options(pythonservicetest PUBLIC -Wno-unknown-pragmas) + target_link_libraries(pythonparsertest ${GTEST_BOTH_LIBRARIES} pthread @@ -20,6 +31,14 @@ target_link_libraries(pythonparsertest model pythonmodel) +target_link_libraries(pythonservicetest + ${GTEST_BOTH_LIBRARIES} + pthread + util + model + pythonmodel + pythonservice) + # Clean up the build folder and the output of the test in a `make clean`. set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES @@ -37,3 +56,13 @@ add_test(NAME pythonparser COMMAND pythonparsertest --workspace ${CMAKE_CURRENT_BINARY_DIR}/workdir/ \ --force" "${TEST_DB}") + +add_test(NAME pythonservice COMMAND pythonservicetest + "echo \"Test database used: ${TEST_DB}\" && \ + ${CMAKE_INSTALL_PREFIX}/bin/CodeCompass_parser \ + --database \"${TEST_DB}\" \ + --name pythonservicetest \ + --input ${CMAKE_CURRENT_SOURCE_DIR}/sources/ \ + --workspace ${CMAKE_CURRENT_BINARY_DIR}/workdir/ \ + --force" + "${TEST_DB}") diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp index 1ea37334e..7fadb4ac5 100644 --- a/plugins/python/test/src/pythonparsertest.cpp +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -15,7 +15,9 @@ class PythonParserTest : public ::testing::Test public: PythonParserTest() : _db(util::connectDatabase(dbConnectionString)), - _transaction(_db) {} + _transaction(_db) + { + } protected: std::shared_ptr _db; diff --git a/plugins/python/test/src/pythonservicetest.cpp b/plugins/python/test/src/pythonservicetest.cpp new file mode 100644 index 000000000..2af3ea187 --- /dev/null +++ b/plugins/python/test/src/pythonservicetest.cpp @@ -0,0 +1,63 @@ +#include +#include +#include +#include + +#include +#include + +using namespace cc; +using namespace cc::service::language; +extern const char* dbConnectionString; + +class PythonServiceTest : public ::testing::Test +{ +public: + PythonServiceTest() : + _db(util::connectDatabase(dbConnectionString)), + _transaction(_db), + _pythonservice(new PythonServiceHandler( + _db, + std::make_shared(), + webserver::ServerContext(std::string(), boost::program_options::variables_map())) + ) + { + } + + model::File queryFile(const std::string& filename) + { + return _transaction([&, this]() { + return _db->query_value(odb::query::filename == filename); + }); + } + + AstNodeInfo getAstNodeInfoByPosition(const std::string& filename, int32_t line, int32_t column) + { + model::File file = queryFile(filename); + + service::core::FilePosition filePos; + filePos.file = std::to_string(file.id); + filePos.pos.line = line; + filePos.pos.column = column; + + AstNodeInfo nodeInfo; + _pythonservice->getAstNodeInfoByPosition(nodeInfo, filePos); + return nodeInfo; + } + +private: + std::shared_ptr _db; + util::OdbTransaction _transaction; + +protected: + std::shared_ptr _pythonservice; +}; + +TEST_F(PythonServiceTest, AstNodeInfoByPosition) +{ + // Simulating click on line 1 column 5 + AstNodeInfo nodeInfo = getAstNodeInfoByPosition("functions.py", 1, 5); + + EXPECT_EQ(nodeInfo.astNodeValue, "def hello_world()"); + EXPECT_EQ(nodeInfo.symbolType, "function"); +} From 0c4d40fe628334d3253ddd4d165cf492853d47f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 18 Nov 2024 14:31:27 +0100 Subject: [PATCH 132/156] [Webgui] Fix caller infinite grouping --- plugins/python/webgui/js/pythonInfoTree.js | 64 +--------------------- 1 file changed, 2 insertions(+), 62 deletions(-) diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index 8b4910c1b..98d376b5f 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -90,8 +90,6 @@ require([ var res = []; var fileGroupsId = []; - scratch = scratch || {}; - var references = model.pythonservice.getReferences( nodeInfo.id, parentNode.refType); @@ -114,11 +112,6 @@ require([ var fileInfo = model.project.getFileInfo(fileId); - if (parentNode.refType === refTypes['Caller']) { - scratch.visitedNodeIDs = - (scratch.visitedNodeIDs || []).concat(nodeInfo.id); - } - res.push({ id : fileGroupsId[fileId], name : createReferenceCountLabel( @@ -131,60 +124,8 @@ require([ var res = []; referenceInFile.forEach(function (reference) { - if (parentNode.refType === refTypes['Caller']) { - var showChildren = - scratch.visitedNodeIDs.indexOf(reference.id) == -1; - res.push({ - id : reference.id, - name : createLabel(reference), - nodeInfo : reference, - refType : parentNode.refType, - cssClass : 'icon icon-Method', - hasChildren : showChildren, - getChildren : showChildren - ? function () { - var res = []; - - //--- Recursive Node ---// - - var refCount = model.pythonservice.getReferenceCount( - reference.id, parentNode.refType); - - if (refCount) - res.push({ - id : 'Caller-' + reference.id, - name : createReferenceCountLabel( - parentNode.name, refCount), - nodeInfo : reference, - refType : parentNode.refType, - cssClass : parentNode.cssClass, - hasChildren : true, - getChildren : function () { - return loadReferenceNodes(this, reference, refTypes, scratch); - } - }); - - //--- Call ---// - - var calls = model.pythonservice.getReferences( - this.nodeInfo.id, - refTypes['This calls']); - - calls.forEach(function (call) { - if (call.entityHash === nodeInfo.entityHash) - res.push({ - name : createLabel(call), - refType : parentNode.refType, - nodeInfo : call, - hasChildren : false, - cssClass : getCssClass(call) - }); - }); - return res; - } - : undefined - }); - } else if (parentNode.refType === refTypes['Usage']) { + if (parentNode.refType === refTypes['Caller'] || + parentNode.refType === refTypes['Usage']) { res.push({ id : fileGroupsId[fileId] + reference.id, name : createLabel(reference), @@ -347,4 +288,3 @@ require([ service : model.pythonservice }); }); - \ No newline at end of file From d02a8ea92cc3c94a9ea3ebff2ae8df9844b1a3f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 18 Nov 2024 15:36:44 +0100 Subject: [PATCH 133/156] [Webgui] Cleanup --- plugins/python/webgui/js/pythonInfoTree.js | 69 ++-------------------- 1 file changed, 6 insertions(+), 63 deletions(-) diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index 98d376b5f..f2af28497 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -6,30 +6,6 @@ require([ model.addService('pythonservice', 'PythonService', LanguageServiceClient); - function createTagLabels(tags) { - var label = ''; - - if (!tags) - return label; - - if (tags.indexOf('static') > -1) - label += 'S'; - if (tags.indexOf('constructor') > -1) - label += 'C'; - if (tags.indexOf('destructor') > -1) - label += 'D'; - if (tags.indexOf('implicit') > -1) - label += 'I'; - if (tags.indexOf('inherited') > -1) - label += 'I'; - if (tags.indexOf('virtual') > -1) - label += 'V'; - if (tags.indexOf('global') > -1) - label += 'G'; - - return label; - } - function createReferenceCountLabel(label, count) { var parsedLabel = $('
').append($.parseHTML(label)); parsedLabel.children('span.reference-count').remove(); @@ -39,36 +15,12 @@ require([ } function createLabel(astNodeInfo) { - var labelClass = ''; - - if (astNodeInfo.tags.indexOf('implicit') > -1) - labelClass = 'label-implicit'; - var labelValue = astNodeInfo.astNodeValue.trim(); if (labelValue.slice(-1) == ':') labelValue = labelValue.substr(0, labelValue.length - 1); if(astNodeInfo.astNodeType && !labelValue.includes(':')) labelValue += ' : ' + astNodeInfo.astNodeType + ""; - // Create dom node for return type of a function and place it at the end of - // signature. - if (astNodeInfo.symbolType === 'Function') { - var init = labelValue.slice(0, labelValue.indexOf('(')); - var returnTypeEnd = init.lastIndexOf(' '); - - //--- Constructor, destructor doesn't have return type ---// - - if (returnTypeEnd !== -1) { - var funcSignature = init.slice(returnTypeEnd); - - labelValue = funcSignature - + ' : ' - + init.slice(0, returnTypeEnd) - + ""; - } - } - - var label = createTagLabels(astNodeInfo.tags) - + '' + var label = '' + astNodeInfo.range.range.startpos.line + ':' + astNodeInfo.range.range.startpos.column + ': ' + labelValue @@ -77,15 +29,6 @@ require([ return label; } - function getCssClass(astNodeInfo) { - var tags = astNodeInfo.tags; - - return tags.indexOf('public') > -1 ? 'icon-visibility icon-public' : - tags.indexOf('private') > -1 ? 'icon-visibility icon-private' : - tags.indexOf('protected') > -1 ? 'icon-visibility icon-protected' : - null; - } - function loadReferenceNodes(parentNode, nodeInfo, refTypes, scratch) { var res = []; var fileGroupsId = []; @@ -132,7 +75,7 @@ require([ refType : parentNode.refType, nodeInfo : reference, hasChildren : false, - cssClass : getCssClass(reference) + cssClass : null }); } }); @@ -145,7 +88,7 @@ require([ refType : parentNode.refType, nodeInfo : reference, hasChildren : false, - cssClass : getCssClass(reference) + cssClass : null }); } }); @@ -170,7 +113,7 @@ require([ refType : parentNode.refType, nodeInfo : reference, hasChildren : false, - cssClass : getCssClass(reference) + cssClass : null }); }); @@ -192,8 +135,8 @@ require([ : elementInfo.name) + ''; - var label = createTagLabels(elementInfo.tags) - + '' + var label = + '' + rootLabel + ': ' + rootValue + ''; From 8a03091578cd50362a3df12e739be8db4fe35bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 18 Nov 2024 15:44:49 +0100 Subject: [PATCH 134/156] [PythonParser] Remove accept function --- plugins/python/parser/include/pythonparser/pythonparser.h | 1 - plugins/python/parser/src/pythonparser.cpp | 6 ------ 2 files changed, 7 deletions(-) diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index f9f0e2f8b..b3b1c25d8 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -33,7 +33,6 @@ class PythonParser : public AbstractParser }; python::object m_py_module; - bool accept(const std::string& path_); void processFile(const python::object& obj, PYNameMap& map, ParseResultStats& parse_result); void parseProject(const std::string& root_path); }; diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 180cecb19..011ddfb4e 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -97,12 +97,6 @@ void PythonParser::parseProject(const std::string& root_path) LOG(info) << "[pythonparser] Partially parsed files: " << parse_result.partial; } -bool PythonParser::accept(const std::string& path_) -{ - std::string ext = boost::filesystem::extension(path_); - return ext == ".py"; -} - void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseResultStats& parse_result) { try { From ffb2676cbbe1ddde6369f919a53817fb6237387c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Mon, 18 Nov 2024 15:52:56 +0100 Subject: [PATCH 135/156] [PyParser] Refactor getFunctionSignature --- plugins/python/parser/pyparser/asthelper.py | 76 +++++++++++---------- plugins/python/parser/pyparser/pyname.py | 2 +- 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index 71c5d5790..d5f6e2630 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -173,6 +173,31 @@ def __getArgumentAnnotation(self, arg: ast.arg) -> PosInfo | None: else: return None + def __getFunctionSignature(self, func: ast.FunctionDef) -> str: + sign = "def " + func.name + sign += "(" + + first = True + for arg in func.args.args: + if first: + first = False + else: + sign += ", " + + sign += arg.arg + + param_annotation: PosInfo | None = self.__getArgumentAnnotation(arg) + if param_annotation: + sign += ": " + param_annotation.value + + sign += ")" + + return_annotation: PosInfo | None = self.__getFunctionReturnAnnotation(func) + if return_annotation: + sign += " -> " + return_annotation.value + + return sign + def getAnnotations(self): if not (self.config.ast and self.config.ast_annotations): return [] @@ -193,24 +218,25 @@ def getAnnotations(self): funchash = getHashName(self.path, funcpos) subhash = getHashName(self.path, subpos) - nodeinfo = NodeInfo() - nodeinfo.id = subhash - nodeinfo.ref_id = subhash - nodeinfo.parent = funchash - nodeinfo.parent_function = funchash - nodeinfo.line_start = subpos.line_start - nodeinfo.line_end = subpos.line_end - nodeinfo.column_start = subpos.column_start - nodeinfo.column_end = subpos.column_end - nodeinfo.file_id = self.file_id - nodeinfo.type = "annotation" - nodeinfo.value = subpos.value + nodeinfo = NodeInfo( + id = subhash, + ref_id = subhash, + parent = funchash, + parent_function = funchash, + line_start = subpos.line_start, + line_end = subpos.line_end, + column_start = subpos.column_start, + column_end = subpos.column_end, + file_id = self.file_id, + type = "annotation", + value = subpos.value + ) results.append(nodeinfo) return results - def getFunctionSignature(self, pos: PosInfo) -> str | None: + def getFunctionSignatureByPosition(self, pos: PosInfo) -> str | None: if not (self.config.ast_function_signature): return None @@ -227,28 +253,6 @@ def getFunctionSignature(self, pos: PosInfo) -> str | None: func.end_col_offset == pos.column_end): continue - sign = "def " + func.name - sign += "(" - - first = True - for arg in func.args.args: - if first: - first = False - else: - sign += ", " - - sign += arg.arg - - param_annotation: PosInfo | None = self.__getArgumentAnnotation(arg) - if param_annotation: - sign += ": " + param_annotation.value - - sign += ")" - - return_annotation: PosInfo | None = self.__getFunctionReturnAnnotation(func) - if return_annotation: - sign += " -> " + return_annotation.value - - return sign + return self.__getFunctionSignature(func) return None diff --git a/plugins/python/parser/pyparser/pyname.py b/plugins/python/parser/pyparser/pyname.py index 3d197bf1a..c12862a43 100644 --- a/plugins/python/parser/pyparser/pyname.py +++ b/plugins/python/parser/pyparser/pyname.py @@ -115,7 +115,7 @@ def getNodeInfo(self) -> NodeInfo: node.value = param if node.type == "function" and node.is_definition and not (node.is_import or node.is_builtin): - sign = self.asthelper.getFunctionSignature(self.pos) + sign = self.asthelper.getFunctionSignatureByPosition(self.pos) node.value = sign if sign else node.value subclass = self.asthelper.getSubclass(self.pos) From 25e9e71b5084a83fe87330f972f7dccb9b003bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 20 Nov 2024 11:32:21 +0100 Subject: [PATCH 136/156] [PyParser] Fix annotation overwritten by jedi node --- plugins/python/parser/pyparser/parser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 5b4af6fc8..770126780 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -128,5 +128,8 @@ def parse(path: str, config: ParserConfig): return result def putInMap(hashmap: dict[int, NodeInfo], node: NodeInfo): + if node.id in hashmap: + return + hashmap[node.id] = node From 137719f18865142b3f7df4b85fa13ee24d2ff765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 20 Nov 2024 11:40:04 +0100 Subject: [PATCH 137/156] [PythonTest] Parser test - functions --- plugins/python/test/sources/functions.py | 89 +++++++++ plugins/python/test/src/pythonparsertest.cpp | 185 ++++++++++++++++++- 2 files changed, 273 insertions(+), 1 deletion(-) diff --git a/plugins/python/test/sources/functions.py b/plugins/python/test/sources/functions.py index df2cf28b6..a03c62b65 100644 --- a/plugins/python/test/sources/functions.py +++ b/plugins/python/test/sources/functions.py @@ -2,3 +2,92 @@ def hello_world(): print("Hello, world!") hello_world() + +def runner(func, param1, param2): + return func(param1,param2) + +def mul(a, b): + return a * b + +def mul2(a, b): + return mul(a,b) + +def mul3(): + return mul + +mylib = { + "multiply": mul +} + +class MyLib: + def __init__(self): + self.multiply = mul + +mylib2 = MyLib() + +a = mul(4,8) +a2 = runner(mul,4,8) +a3 = mul3()(4,8) +a4 = mylib["multiply"](4,8) +a5 = mylib2.multiply(4,8) + +print(a) +print(a2) +print(a3) +print(a4) +print(a5) + +print("----------") + +print(mul(4,8)) +print(runner(mul,4,8)) +print(mul3()(4,8)) +print(mylib["multiply"](4,8)) +print(mylib2.multiply(4,8)) + +def sign(a: int, b: str) -> None: + pass + +def sign2( + a: int, + b: str) -> None: + pass + +def sign3( + a: int = 2, + b: str = "hi") -> None: + pass + +def sign4( # note + a: int = 2, # note 2 + b: str = "hi") -> None: # note 3 + pass + +from typing import List, Optional + +def annotation(a, b) -> None: + pass + +def annotation2(a, + b) -> str: + return "abc" + +def annotation3(a, + b) -> int: + return 0 + +def annotation4(a, + b) -> bool: + return True + +def annotation5(a, + b) -> List[str]: + return [] + +def annotation6(a, + b) -> Optional[str]: + pass + +def annotation7(a, + b) -> dict[int, bool]: + return {} diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp index 7fadb4ac5..96a7b5ff9 100644 --- a/plugins/python/test/src/pythonparsertest.cpp +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -35,12 +35,195 @@ TEST_F(PythonParserTest, FilesAreInDatabase) }); } -TEST_F(PythonParserTest, FunctionDefinitions) +TEST_F(PythonParserTest, FunctionDefinition) { _transaction([&, this]() { model::PYName pyname; pyname = _db->query_value(odb::query::value == "def hello_world()"); EXPECT_EQ(pyname.is_definition, true); + + pyname = _db->query_value(odb::query::value == "def runner(func, param1, param2)"); + EXPECT_EQ(pyname.is_definition, true); + + pyname = _db->query_value(odb::query::value == "def mul(a, b)"); + EXPECT_EQ(pyname.is_definition, true); + + pyname = _db->query_value(odb::query::value == "def mul2(a, b)"); + EXPECT_EQ(pyname.is_definition, true); + + pyname = _db->query_value(odb::query::value == "def mul3()"); + EXPECT_EQ(pyname.is_definition, true); + }); +} + +TEST_F(PythonParserTest, FunctionType) +{ + _transaction([&, this]() { + model::PYName pyname; + + pyname = _db->query_value(odb::query::value == "def hello_world()"); + EXPECT_EQ(pyname.type, "function"); + + pyname = _db->query_value(odb::query::value == "def runner(func, param1, param2)"); + EXPECT_EQ(pyname.type, "function"); + + pyname = _db->query_value(odb::query::value == "def mul(a, b)"); + EXPECT_EQ(pyname.type, "function"); + + pyname = _db->query_value(odb::query::value == "def mul2(a, b)"); + EXPECT_EQ(pyname.type, "function"); + + pyname = _db->query_value(odb::query::value == "def mul3()"); + EXPECT_EQ(pyname.type, "function"); + }); +} + +TEST_F(PythonParserTest, FunctionParamAST) +{ + _transaction([&, this]() { + model::PYName pyname; + + pyname = _db->query_value(odb::query::line_start == 6 && + odb::query::column_start == 12); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = _db->query_value(odb::query::line_start == 6 && + odb::query::column_start == 18); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = _db->query_value(odb::query::line_start == 6 && + odb::query::column_start == 26); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = _db->query_value(odb::query::line_start == 9 && + odb::query::column_start == 9); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = _db->query_value(odb::query::line_start == 9 && + odb::query::column_start == 12); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = _db->query_value(odb::query::line_start == 12 && + odb::query::column_start == 10); + EXPECT_EQ(pyname.type, "astparam"); + + pyname = _db->query_value(odb::query::line_start == 12 && + odb::query::column_start == 13); + EXPECT_EQ(pyname.type, "astparam"); + }); +} + +TEST_F(PythonParserTest, FunctionSignatureAST) +{ + _transaction([&, this]() { + model::PYName pyname; + + pyname = _db->query_value(odb::query::line_start == 48 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign(a: int, b: str) -> None"); + + pyname = _db->query_value(odb::query::line_start == 51 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign2(a: int, b: str) -> None"); + + pyname = _db->query_value(odb::query::line_start == 56 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign3(a: int, b: str) -> None"); + + pyname = _db->query_value(odb::query::line_start == 61 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign4(a: int, b: str) -> None"); + }); +} + +TEST_F(PythonParserTest, FunctionAnnotationAST) +{ + _transaction([&, this]() { + model::PYName pyname; + + pyname = _db->query_value(odb::query::line_start == 68 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "None"); + + pyname = _db->query_value(odb::query::line_start == 72 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "str"); + + pyname = _db->query_value(odb::query::line_start == 76 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "int"); + + pyname = _db->query_value(odb::query::line_start == 80 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "bool"); + + pyname = _db->query_value(odb::query::line_start == 84 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "List[str]"); + + pyname = _db->query_value(odb::query::line_start == 88 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "Optional[str]"); + + pyname = _db->query_value(odb::query::line_start == 92 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "dict[int, bool]"); + }); +} + +TEST_F(PythonParserTest, FunctionCall) +{ + _transaction([&, this]() { + model::PYName pyname; + + pyname = _db->query_value(odb::query::value == "def hello_world()"); + EXPECT_EQ(pyname.is_call, false); + + pyname = _db->query_value(odb::query::value == "def runner(func, param1, param2)"); + EXPECT_EQ(pyname.is_call, false); + + pyname = _db->query_value(odb::query::value == "def mul(a, b)"); + EXPECT_EQ(pyname.is_call, false); + + pyname = _db->query_value(odb::query::value == "def mul2(a, b)"); + EXPECT_EQ(pyname.is_call, false); + + pyname = _db->query_value(odb::query::value == "def mul3()"); + EXPECT_EQ(pyname.is_call, false); + + // ---------- + + pyname = _db->query_value(odb::query::value == "mul" && + odb::query::line_start == 13); + EXPECT_EQ(pyname.is_call, true); + + pyname = _db->query_value(odb::query::value == "mul" && + odb::query::line_start == 16); + EXPECT_EQ(pyname.is_call, false); + + pyname = _db->query_value(odb::query::value == "mul" && + odb::query::line_start == 19); + EXPECT_EQ(pyname.is_call, false); + + pyname = _db->query_value(odb::query::value == "mul" && + odb::query::line_start == 24); + EXPECT_EQ(pyname.is_call, false); + + pyname = _db->query_value(odb::query::value == "mul" && + odb::query::line_start == 28); + EXPECT_EQ(pyname.is_call, true); + + pyname = _db->query_value(odb::query::value == "mul" && + odb::query::line_start == 29); + EXPECT_EQ(pyname.is_call, false); + + pyname = _db->query_value(odb::query::value == "mul" && + odb::query::line_start == 42); + EXPECT_EQ(pyname.is_call, true); + + pyname = _db->query_value(odb::query::value == "mul" && + odb::query::line_start == 43); + EXPECT_EQ(pyname.is_call, false); }); } From 08dd0a80aeea227f23d16922c78374db0e3d869f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 20 Nov 2024 13:24:43 +0100 Subject: [PATCH 138/156] [PythonTest] Parser test - class inheritance --- plugins/python/test/sources/classes.py | 23 +++++++++++ plugins/python/test/src/pythonparsertest.cpp | 43 ++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 plugins/python/test/sources/classes.py diff --git a/plugins/python/test/sources/classes.py b/plugins/python/test/sources/classes.py new file mode 100644 index 000000000..1503c00f1 --- /dev/null +++ b/plugins/python/test/sources/classes.py @@ -0,0 +1,23 @@ +class Base: + DEBUG_MODE = False + users = [] + + def __init__(self) -> None: + pass + + def foo(self): + pass + + def bar(self): + def test(): + pass + + test() + +class Derived(Base): + def __init__(self) -> None: + pass + +class Derived2(Derived, Base): + def __init__(self) -> None: + pass diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp index 96a7b5ff9..d754a38e1 100644 --- a/plugins/python/test/src/pythonparsertest.cpp +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -227,3 +227,46 @@ TEST_F(PythonParserTest, FunctionCall) EXPECT_EQ(pyname.is_call, false); }); } + +TEST_F(PythonParserTest, ClassInheritance) +{ + _transaction([&, this]() { + model::PYName pyname; + + model::PYName base = _db->query_value + (odb::query::line_start == 1 && + odb::query::type == "class"); + + model::PYName derived = _db->query_value + (odb::query::line_start == 17 && + odb::query::type == "class"); + + model::PYName derived2 = _db->query_value + (odb::query::line_start == 21 && + odb::query::type == "class"); + + pyname = _db->query_value + (odb::query::line_start == 17 && + odb::query::column_start == 15 && + odb::query::value == "Base"); + + EXPECT_EQ(pyname.type, "baseclass"); + EXPECT_EQ(pyname.parent, derived.id); + + pyname = _db->query_value + (odb::query::line_start == 21 && + odb::query::column_start == 16 && + odb::query::value == "Derived"); + + EXPECT_EQ(pyname.type, "baseclass"); + EXPECT_EQ(pyname.parent, derived2.id); + + pyname = _db->query_value + (odb::query::line_start == 21 && + odb::query::column_start == 25 && + odb::query::value == "Base"); + + EXPECT_EQ(pyname.type, "baseclass"); + EXPECT_EQ(pyname.parent, derived2.id); + }); +} From b8ed0936f8acb69dcadf16208534adf4475ad353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 20 Nov 2024 13:52:25 +0100 Subject: [PATCH 139/156] [PythonTest] Parser test - queryFile --- plugins/python/test/src/pythonparsertest.cpp | 327 ++++++++++--------- 1 file changed, 170 insertions(+), 157 deletions(-) diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp index d754a38e1..d5109d233 100644 --- a/plugins/python/test/src/pythonparsertest.cpp +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -1,7 +1,9 @@ +#include #include #include #include +#include #include #include #include @@ -17,6 +19,31 @@ class PythonParserTest : public ::testing::Test _db(util::connectDatabase(dbConnectionString)), _transaction(_db) { + loadFile("functions.py"); + loadFile("classes.py"); + } + + model::PYName queryFile(const std::string& filename, const odb::query& odb_query) + { + model::PYName pyname; + if (m_files.count(filename)) + { + _transaction([&, this]() { + pyname = _db->query_value(odb_query && odb::query::file_id == m_files[filename]); + }); + } + + return pyname; + } + +private: + std::unordered_map m_files; + void loadFile(const std::string& filename) + { + _transaction([&, this]() { + model::File file = _db->query_value(odb::query::filename == filename); + m_files.emplace(filename, file.id); + }); } protected: @@ -37,236 +64,222 @@ TEST_F(PythonParserTest, FilesAreInDatabase) TEST_F(PythonParserTest, FunctionDefinition) { - _transaction([&, this]() { - model::PYName pyname; + model::PYName pyname; - pyname = _db->query_value(odb::query::value == "def hello_world()"); - EXPECT_EQ(pyname.is_definition, true); + pyname = queryFile("functions.py", odb::query::value == "def hello_world()"); + EXPECT_EQ(pyname.is_definition, true); - pyname = _db->query_value(odb::query::value == "def runner(func, param1, param2)"); - EXPECT_EQ(pyname.is_definition, true); + pyname = queryFile("functions.py", odb::query::value == "def runner(func, param1, param2)"); + EXPECT_EQ(pyname.is_definition, true); - pyname = _db->query_value(odb::query::value == "def mul(a, b)"); - EXPECT_EQ(pyname.is_definition, true); + pyname = queryFile("functions.py", odb::query::value == "def mul(a, b)"); + EXPECT_EQ(pyname.is_definition, true); - pyname = _db->query_value(odb::query::value == "def mul2(a, b)"); - EXPECT_EQ(pyname.is_definition, true); + pyname = queryFile("functions.py", odb::query::value == "def mul2(a, b)"); + EXPECT_EQ(pyname.is_definition, true); - pyname = _db->query_value(odb::query::value == "def mul3()"); - EXPECT_EQ(pyname.is_definition, true); - }); + pyname = queryFile("functions.py", odb::query::value == "def mul3()"); + EXPECT_EQ(pyname.is_definition, true); } TEST_F(PythonParserTest, FunctionType) { - _transaction([&, this]() { - model::PYName pyname; + model::PYName pyname; - pyname = _db->query_value(odb::query::value == "def hello_world()"); - EXPECT_EQ(pyname.type, "function"); + pyname = queryFile("functions.py", odb::query::value == "def hello_world()"); + EXPECT_EQ(pyname.type, "function"); - pyname = _db->query_value(odb::query::value == "def runner(func, param1, param2)"); - EXPECT_EQ(pyname.type, "function"); + pyname = queryFile("functions.py", odb::query::value == "def runner(func, param1, param2)"); + EXPECT_EQ(pyname.type, "function"); - pyname = _db->query_value(odb::query::value == "def mul(a, b)"); - EXPECT_EQ(pyname.type, "function"); + pyname = queryFile("functions.py", odb::query::value == "def mul(a, b)"); + EXPECT_EQ(pyname.type, "function"); - pyname = _db->query_value(odb::query::value == "def mul2(a, b)"); - EXPECT_EQ(pyname.type, "function"); + pyname = queryFile("functions.py", odb::query::value == "def mul2(a, b)"); + EXPECT_EQ(pyname.type, "function"); - pyname = _db->query_value(odb::query::value == "def mul3()"); - EXPECT_EQ(pyname.type, "function"); - }); + pyname = queryFile("functions.py", odb::query::value == "def mul3()"); + EXPECT_EQ(pyname.type, "function"); } TEST_F(PythonParserTest, FunctionParamAST) { - _transaction([&, this]() { - model::PYName pyname; + model::PYName pyname; - pyname = _db->query_value(odb::query::line_start == 6 && - odb::query::column_start == 12); - EXPECT_EQ(pyname.type, "astparam"); + pyname = queryFile("functions.py", odb::query::line_start == 6 && + odb::query::column_start == 12); + EXPECT_EQ(pyname.type, "astparam"); - pyname = _db->query_value(odb::query::line_start == 6 && - odb::query::column_start == 18); - EXPECT_EQ(pyname.type, "astparam"); + pyname = queryFile("functions.py", odb::query::line_start == 6 && + odb::query::column_start == 18); + EXPECT_EQ(pyname.type, "astparam"); - pyname = _db->query_value(odb::query::line_start == 6 && - odb::query::column_start == 26); - EXPECT_EQ(pyname.type, "astparam"); + pyname = queryFile("functions.py", odb::query::line_start == 6 && + odb::query::column_start == 26); + EXPECT_EQ(pyname.type, "astparam"); - pyname = _db->query_value(odb::query::line_start == 9 && - odb::query::column_start == 9); - EXPECT_EQ(pyname.type, "astparam"); + pyname = queryFile("functions.py", odb::query::line_start == 9 && + odb::query::column_start == 9); + EXPECT_EQ(pyname.type, "astparam"); - pyname = _db->query_value(odb::query::line_start == 9 && - odb::query::column_start == 12); - EXPECT_EQ(pyname.type, "astparam"); + pyname = queryFile("functions.py", odb::query::line_start == 9 && + odb::query::column_start == 12); + EXPECT_EQ(pyname.type, "astparam"); - pyname = _db->query_value(odb::query::line_start == 12 && - odb::query::column_start == 10); - EXPECT_EQ(pyname.type, "astparam"); + pyname = queryFile("functions.py", odb::query::line_start == 12 && + odb::query::column_start == 10); + EXPECT_EQ(pyname.type, "astparam"); - pyname = _db->query_value(odb::query::line_start == 12 && - odb::query::column_start == 13); - EXPECT_EQ(pyname.type, "astparam"); - }); + pyname = queryFile("functions.py", odb::query::line_start == 12 && + odb::query::column_start == 13); + EXPECT_EQ(pyname.type, "astparam"); } TEST_F(PythonParserTest, FunctionSignatureAST) { - _transaction([&, this]() { - model::PYName pyname; + model::PYName pyname; - pyname = _db->query_value(odb::query::line_start == 48 && - odb::query::type == "function"); - EXPECT_EQ(pyname.value, "def sign(a: int, b: str) -> None"); + pyname = queryFile("functions.py", odb::query::line_start == 48 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign(a: int, b: str) -> None"); - pyname = _db->query_value(odb::query::line_start == 51 && - odb::query::type == "function"); - EXPECT_EQ(pyname.value, "def sign2(a: int, b: str) -> None"); + pyname = queryFile("functions.py", odb::query::line_start == 51 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign2(a: int, b: str) -> None"); - pyname = _db->query_value(odb::query::line_start == 56 && - odb::query::type == "function"); - EXPECT_EQ(pyname.value, "def sign3(a: int, b: str) -> None"); + pyname = queryFile("functions.py", odb::query::line_start == 56 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign3(a: int, b: str) -> None"); - pyname = _db->query_value(odb::query::line_start == 61 && - odb::query::type == "function"); - EXPECT_EQ(pyname.value, "def sign4(a: int, b: str) -> None"); - }); + pyname = queryFile("functions.py", odb::query::line_start == 61 && + odb::query::type == "function"); + EXPECT_EQ(pyname.value, "def sign4(a: int, b: str) -> None"); } TEST_F(PythonParserTest, FunctionAnnotationAST) { - _transaction([&, this]() { - model::PYName pyname; + model::PYName pyname; - pyname = _db->query_value(odb::query::line_start == 68 && - odb::query::type == "annotation"); - EXPECT_EQ(pyname.value, "None"); + pyname = queryFile("functions.py", odb::query::line_start == 68 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "None"); - pyname = _db->query_value(odb::query::line_start == 72 && - odb::query::type == "annotation"); - EXPECT_EQ(pyname.value, "str"); + pyname = queryFile("functions.py", odb::query::line_start == 72 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "str"); - pyname = _db->query_value(odb::query::line_start == 76 && - odb::query::type == "annotation"); - EXPECT_EQ(pyname.value, "int"); + pyname = queryFile("functions.py", odb::query::line_start == 76 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "int"); - pyname = _db->query_value(odb::query::line_start == 80 && - odb::query::type == "annotation"); - EXPECT_EQ(pyname.value, "bool"); + pyname = queryFile("functions.py", odb::query::line_start == 80 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "bool"); - pyname = _db->query_value(odb::query::line_start == 84 && - odb::query::type == "annotation"); - EXPECT_EQ(pyname.value, "List[str]"); + pyname = queryFile("functions.py", odb::query::line_start == 84 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "List[str]"); - pyname = _db->query_value(odb::query::line_start == 88 && - odb::query::type == "annotation"); - EXPECT_EQ(pyname.value, "Optional[str]"); + pyname = queryFile("functions.py", odb::query::line_start == 88 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "Optional[str]"); - pyname = _db->query_value(odb::query::line_start == 92 && - odb::query::type == "annotation"); - EXPECT_EQ(pyname.value, "dict[int, bool]"); - }); + pyname = queryFile("functions.py", odb::query::line_start == 92 && + odb::query::type == "annotation"); + EXPECT_EQ(pyname.value, "dict[int, bool]"); } TEST_F(PythonParserTest, FunctionCall) { - _transaction([&, this]() { - model::PYName pyname; + model::PYName pyname; - pyname = _db->query_value(odb::query::value == "def hello_world()"); - EXPECT_EQ(pyname.is_call, false); + pyname = queryFile("functions.py", odb::query::value == "def hello_world()"); + EXPECT_EQ(pyname.is_call, false); - pyname = _db->query_value(odb::query::value == "def runner(func, param1, param2)"); - EXPECT_EQ(pyname.is_call, false); + pyname = queryFile("functions.py", odb::query::value == "def runner(func, param1, param2)"); + EXPECT_EQ(pyname.is_call, false); - pyname = _db->query_value(odb::query::value == "def mul(a, b)"); - EXPECT_EQ(pyname.is_call, false); + pyname = queryFile("functions.py", odb::query::value == "def mul(a, b)"); + EXPECT_EQ(pyname.is_call, false); - pyname = _db->query_value(odb::query::value == "def mul2(a, b)"); - EXPECT_EQ(pyname.is_call, false); + pyname = queryFile("functions.py", odb::query::value == "def mul2(a, b)"); + EXPECT_EQ(pyname.is_call, false); - pyname = _db->query_value(odb::query::value == "def mul3()"); - EXPECT_EQ(pyname.is_call, false); + pyname = queryFile("functions.py", odb::query::value == "def mul3()"); + EXPECT_EQ(pyname.is_call, false); - // ---------- + // ---------- - pyname = _db->query_value(odb::query::value == "mul" && - odb::query::line_start == 13); - EXPECT_EQ(pyname.is_call, true); + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 13); + EXPECT_EQ(pyname.is_call, true); - pyname = _db->query_value(odb::query::value == "mul" && - odb::query::line_start == 16); - EXPECT_EQ(pyname.is_call, false); + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 16); + EXPECT_EQ(pyname.is_call, false); - pyname = _db->query_value(odb::query::value == "mul" && - odb::query::line_start == 19); - EXPECT_EQ(pyname.is_call, false); + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 19); + EXPECT_EQ(pyname.is_call, false); - pyname = _db->query_value(odb::query::value == "mul" && - odb::query::line_start == 24); - EXPECT_EQ(pyname.is_call, false); + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 24); + EXPECT_EQ(pyname.is_call, false); - pyname = _db->query_value(odb::query::value == "mul" && - odb::query::line_start == 28); - EXPECT_EQ(pyname.is_call, true); + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 28); + EXPECT_EQ(pyname.is_call, true); - pyname = _db->query_value(odb::query::value == "mul" && - odb::query::line_start == 29); - EXPECT_EQ(pyname.is_call, false); + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 29); + EXPECT_EQ(pyname.is_call, false); - pyname = _db->query_value(odb::query::value == "mul" && - odb::query::line_start == 42); - EXPECT_EQ(pyname.is_call, true); + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 42); + EXPECT_EQ(pyname.is_call, true); - pyname = _db->query_value(odb::query::value == "mul" && - odb::query::line_start == 43); - EXPECT_EQ(pyname.is_call, false); - }); + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 43); + EXPECT_EQ(pyname.is_call, false); } TEST_F(PythonParserTest, ClassInheritance) { - _transaction([&, this]() { - model::PYName pyname; + model::PYName pyname; - model::PYName base = _db->query_value - (odb::query::line_start == 1 && - odb::query::type == "class"); + model::PYName base = queryFile("classes.py", + (odb::query::line_start == 1 && + odb::query::type == "class")); - model::PYName derived = _db->query_value - (odb::query::line_start == 17 && - odb::query::type == "class"); + model::PYName derived = queryFile("classes.py", + (odb::query::line_start == 17 && + odb::query::type == "class")); - model::PYName derived2 = _db->query_value - (odb::query::line_start == 21 && - odb::query::type == "class"); + model::PYName derived2 = queryFile("classes.py", + (odb::query::line_start == 21 && + odb::query::type == "class")); - pyname = _db->query_value - (odb::query::line_start == 17 && - odb::query::column_start == 15 && - odb::query::value == "Base"); + pyname = queryFile("classes.py", + (odb::query::line_start == 17 && + odb::query::column_start == 15 && + odb::query::value == "Base")); - EXPECT_EQ(pyname.type, "baseclass"); - EXPECT_EQ(pyname.parent, derived.id); + EXPECT_EQ(pyname.type, "baseclass"); + EXPECT_EQ(pyname.parent, derived.id); - pyname = _db->query_value - (odb::query::line_start == 21 && - odb::query::column_start == 16 && - odb::query::value == "Derived"); + pyname = queryFile("classes.py", + (odb::query::line_start == 21 && + odb::query::column_start == 16 && + odb::query::value == "Derived")); - EXPECT_EQ(pyname.type, "baseclass"); - EXPECT_EQ(pyname.parent, derived2.id); + EXPECT_EQ(pyname.type, "baseclass"); + EXPECT_EQ(pyname.parent, derived2.id); - pyname = _db->query_value - (odb::query::line_start == 21 && - odb::query::column_start == 25 && - odb::query::value == "Base"); + pyname = queryFile("classes.py", + (odb::query::line_start == 21 && + odb::query::column_start == 25 && + odb::query::value == "Base")); - EXPECT_EQ(pyname.type, "baseclass"); - EXPECT_EQ(pyname.parent, derived2.id); - }); + EXPECT_EQ(pyname.type, "baseclass"); + EXPECT_EQ(pyname.parent, derived2.id); } From ed939a3753afcc8780ad7bb641ad9d66ed195c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 20 Nov 2024 14:36:50 +0100 Subject: [PATCH 140/156] [PythonTest] Parser test - class methods, local variables --- plugins/python/test/sources/functions.py | 5 ++ plugins/python/test/src/pythonparsertest.cpp | 77 +++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/plugins/python/test/sources/functions.py b/plugins/python/test/sources/functions.py index a03c62b65..67073d5b5 100644 --- a/plugins/python/test/sources/functions.py +++ b/plugins/python/test/sources/functions.py @@ -91,3 +91,8 @@ def annotation6(a, def annotation7(a, b) -> dict[int, bool]: return {} + +def local_var(): + a = 2 + for i in range(0,10): + a += i diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp index d5109d233..080f72c06 100644 --- a/plugins/python/test/src/pythonparsertest.cpp +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -247,10 +247,6 @@ TEST_F(PythonParserTest, ClassInheritance) { model::PYName pyname; - model::PYName base = queryFile("classes.py", - (odb::query::line_start == 1 && - odb::query::type == "class")); - model::PYName derived = queryFile("classes.py", (odb::query::line_start == 17 && odb::query::type == "class")); @@ -283,3 +279,76 @@ TEST_F(PythonParserTest, ClassInheritance) EXPECT_EQ(pyname.type, "baseclass"); EXPECT_EQ(pyname.parent, derived2.id); } + +TEST_F(PythonParserTest, ClassMethod) +{ + model::PYName base = queryFile("classes.py", + (odb::query::line_start == 1 && + odb::query::type == "class")); + + model::PYName foo = queryFile("classes.py", + (odb::query::line_start == 8 && + odb::query::is_definition == true && + odb::query::type == "function")); + + EXPECT_EQ(foo.parent, base.id); + + model::PYName bar = queryFile("classes.py", + (odb::query::line_start == 11 && + odb::query::is_definition == true && + odb::query::type == "function")); + + EXPECT_EQ(bar.parent, base.id); + + model::PYName test = queryFile("classes.py", + (odb::query::line_start == 12 && + odb::query::is_definition == true && + odb::query::type == "function")); + + EXPECT_EQ(test.parent, bar.id); +} + +TEST_F(PythonParserTest, ClassDataMember) +{ + model::PYName pyname; + + model::PYName base = queryFile("classes.py", + (odb::query::line_start == 1 && + odb::query::type == "class")); + + pyname = queryFile("classes.py", + odb::query::value == "DEBUG_MODE = False"); + + EXPECT_EQ(pyname.type, "statement"); + EXPECT_EQ(pyname.parent, base.id); + + pyname = queryFile("classes.py", + odb::query::value == "users = []"); + + EXPECT_EQ(pyname.type, "statement"); + EXPECT_EQ(pyname.parent, base.id); +} + +TEST_F(PythonParserTest, LocalVariable) +{ + model::PYName pyname; + + model::PYName func = queryFile("functions.py", + odb::query::value == "def local_var()"); + + pyname = queryFile("functions.py", + (odb::query::line_start == 96 && + odb::query::value == "a = 2")); + + EXPECT_EQ(pyname.type, "statement"); + EXPECT_EQ(pyname.is_definition, true); + EXPECT_EQ(pyname.parent, func.id); + + pyname = queryFile("functions.py", + (odb::query::line_start == 97 && + odb::query::value == "for i in range(0,10):\n")); + + EXPECT_EQ(pyname.type, "statement"); + EXPECT_EQ(pyname.is_definition, true); + EXPECT_EQ(pyname.parent, func.id); +} From 67ead9f1b8cb4d9a2744e64ec51a44ab3e93c248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 20 Nov 2024 15:26:19 +0100 Subject: [PATCH 141/156] [PythonTest] Parser test - imports --- plugins/python/test/sources/classes.py | 2 + plugins/python/test/sources/functions.py | 27 +++--- plugins/python/test/sources/imports.py | 12 +++ plugins/python/test/src/pythonparsertest.cpp | 92 ++++++++++++-------- 4 files changed, 86 insertions(+), 47 deletions(-) create mode 100644 plugins/python/test/sources/imports.py diff --git a/plugins/python/test/sources/classes.py b/plugins/python/test/sources/classes.py index 1503c00f1..b1cf4474a 100644 --- a/plugins/python/test/sources/classes.py +++ b/plugins/python/test/sources/classes.py @@ -9,6 +9,8 @@ def foo(self): pass def bar(self): + print("bar") + def test(): pass diff --git a/plugins/python/test/sources/functions.py b/plugins/python/test/sources/functions.py index 67073d5b5..5fe172291 100644 --- a/plugins/python/test/sources/functions.py +++ b/plugins/python/test/sources/functions.py @@ -1,8 +1,6 @@ def hello_world(): print("Hello, world!") -hello_world() - def runner(func, param1, param2): return func(param1,param2) @@ -31,19 +29,22 @@ def __init__(self): a4 = mylib["multiply"](4,8) a5 = mylib2.multiply(4,8) -print(a) -print(a2) -print(a3) -print(a4) -print(a5) +if __name__ == "__main__": + hello_world() + + print(a) + print(a2) + print(a3) + print(a4) + print(a5) -print("----------") + print("----------") -print(mul(4,8)) -print(runner(mul,4,8)) -print(mul3()(4,8)) -print(mylib["multiply"](4,8)) -print(mylib2.multiply(4,8)) + print(mul(4,8)) + print(runner(mul,4,8)) + print(mul3()(4,8)) + print(mylib["multiply"](4,8)) + print(mylib2.multiply(4,8)) def sign(a: int, b: str) -> None: pass diff --git a/plugins/python/test/sources/imports.py b/plugins/python/test/sources/imports.py new file mode 100644 index 000000000..82b538afb --- /dev/null +++ b/plugins/python/test/sources/imports.py @@ -0,0 +1,12 @@ +import classes +import os +from functions import mul + +a = mul(4,8) +print(a) +print(mul(4,8)) + +base = classes.Base() +base.bar() + +print("pid", os.getpid()) diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp index 080f72c06..c3d6bafcf 100644 --- a/plugins/python/test/src/pythonparsertest.cpp +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -21,6 +21,7 @@ class PythonParserTest : public ::testing::Test { loadFile("functions.py"); loadFile("classes.py"); + loadFile("imports.py"); } model::PYName queryFile(const std::string& filename, const odb::query& odb_query) @@ -106,31 +107,31 @@ TEST_F(PythonParserTest, FunctionParamAST) { model::PYName pyname; - pyname = queryFile("functions.py", odb::query::line_start == 6 && + pyname = queryFile("functions.py", odb::query::line_start == 4 && odb::query::column_start == 12); EXPECT_EQ(pyname.type, "astparam"); - pyname = queryFile("functions.py", odb::query::line_start == 6 && + pyname = queryFile("functions.py", odb::query::line_start == 4 && odb::query::column_start == 18); EXPECT_EQ(pyname.type, "astparam"); - pyname = queryFile("functions.py", odb::query::line_start == 6 && + pyname = queryFile("functions.py", odb::query::line_start == 4 && odb::query::column_start == 26); EXPECT_EQ(pyname.type, "astparam"); - pyname = queryFile("functions.py", odb::query::line_start == 9 && + pyname = queryFile("functions.py", odb::query::line_start == 7 && odb::query::column_start == 9); EXPECT_EQ(pyname.type, "astparam"); - pyname = queryFile("functions.py", odb::query::line_start == 9 && + pyname = queryFile("functions.py", odb::query::line_start == 7 && odb::query::column_start == 12); EXPECT_EQ(pyname.type, "astparam"); - pyname = queryFile("functions.py", odb::query::line_start == 12 && + pyname = queryFile("functions.py", odb::query::line_start == 10 && odb::query::column_start == 10); EXPECT_EQ(pyname.type, "astparam"); - pyname = queryFile("functions.py", odb::query::line_start == 12 && + pyname = queryFile("functions.py", odb::query::line_start == 10 && odb::query::column_start == 13); EXPECT_EQ(pyname.type, "astparam"); } @@ -139,19 +140,19 @@ TEST_F(PythonParserTest, FunctionSignatureAST) { model::PYName pyname; - pyname = queryFile("functions.py", odb::query::line_start == 48 && + pyname = queryFile("functions.py", odb::query::line_start == 49 && odb::query::type == "function"); EXPECT_EQ(pyname.value, "def sign(a: int, b: str) -> None"); - pyname = queryFile("functions.py", odb::query::line_start == 51 && + pyname = queryFile("functions.py", odb::query::line_start == 52 && odb::query::type == "function"); EXPECT_EQ(pyname.value, "def sign2(a: int, b: str) -> None"); - pyname = queryFile("functions.py", odb::query::line_start == 56 && + pyname = queryFile("functions.py", odb::query::line_start == 57 && odb::query::type == "function"); EXPECT_EQ(pyname.value, "def sign3(a: int, b: str) -> None"); - pyname = queryFile("functions.py", odb::query::line_start == 61 && + pyname = queryFile("functions.py", odb::query::line_start == 62 && odb::query::type == "function"); EXPECT_EQ(pyname.value, "def sign4(a: int, b: str) -> None"); } @@ -160,31 +161,31 @@ TEST_F(PythonParserTest, FunctionAnnotationAST) { model::PYName pyname; - pyname = queryFile("functions.py", odb::query::line_start == 68 && + pyname = queryFile("functions.py", odb::query::line_start == 69 && odb::query::type == "annotation"); EXPECT_EQ(pyname.value, "None"); - pyname = queryFile("functions.py", odb::query::line_start == 72 && + pyname = queryFile("functions.py", odb::query::line_start == 73 && odb::query::type == "annotation"); EXPECT_EQ(pyname.value, "str"); - pyname = queryFile("functions.py", odb::query::line_start == 76 && + pyname = queryFile("functions.py", odb::query::line_start == 77 && odb::query::type == "annotation"); EXPECT_EQ(pyname.value, "int"); - pyname = queryFile("functions.py", odb::query::line_start == 80 && + pyname = queryFile("functions.py", odb::query::line_start == 81 && odb::query::type == "annotation"); EXPECT_EQ(pyname.value, "bool"); - pyname = queryFile("functions.py", odb::query::line_start == 84 && + pyname = queryFile("functions.py", odb::query::line_start == 85 && odb::query::type == "annotation"); EXPECT_EQ(pyname.value, "List[str]"); - pyname = queryFile("functions.py", odb::query::line_start == 88 && + pyname = queryFile("functions.py", odb::query::line_start == 89 && odb::query::type == "annotation"); EXPECT_EQ(pyname.value, "Optional[str]"); - pyname = queryFile("functions.py", odb::query::line_start == 92 && + pyname = queryFile("functions.py", odb::query::line_start == 93 && odb::query::type == "annotation"); EXPECT_EQ(pyname.value, "dict[int, bool]"); } @@ -211,35 +212,35 @@ TEST_F(PythonParserTest, FunctionCall) // ---------- pyname = queryFile("functions.py", odb::query::value == "mul" && - odb::query::line_start == 13); + odb::query::line_start == 11); EXPECT_EQ(pyname.is_call, true); pyname = queryFile("functions.py", odb::query::value == "mul" && - odb::query::line_start == 16); + odb::query::line_start == 14); EXPECT_EQ(pyname.is_call, false); pyname = queryFile("functions.py", odb::query::value == "mul" && - odb::query::line_start == 19); + odb::query::line_start == 17); EXPECT_EQ(pyname.is_call, false); pyname = queryFile("functions.py", odb::query::value == "mul" && - odb::query::line_start == 24); + odb::query::line_start == 22); EXPECT_EQ(pyname.is_call, false); pyname = queryFile("functions.py", odb::query::value == "mul" && - odb::query::line_start == 28); + odb::query::line_start == 26); EXPECT_EQ(pyname.is_call, true); pyname = queryFile("functions.py", odb::query::value == "mul" && - odb::query::line_start == 29); + odb::query::line_start == 27); EXPECT_EQ(pyname.is_call, false); pyname = queryFile("functions.py", odb::query::value == "mul" && - odb::query::line_start == 42); + odb::query::line_start == 43); EXPECT_EQ(pyname.is_call, true); pyname = queryFile("functions.py", odb::query::value == "mul" && - odb::query::line_start == 43); + odb::query::line_start == 44); EXPECT_EQ(pyname.is_call, false); } @@ -248,15 +249,15 @@ TEST_F(PythonParserTest, ClassInheritance) model::PYName pyname; model::PYName derived = queryFile("classes.py", - (odb::query::line_start == 17 && + (odb::query::line_start == 19 && odb::query::type == "class")); model::PYName derived2 = queryFile("classes.py", - (odb::query::line_start == 21 && + (odb::query::line_start == 23 && odb::query::type == "class")); pyname = queryFile("classes.py", - (odb::query::line_start == 17 && + (odb::query::line_start == 19 && odb::query::column_start == 15 && odb::query::value == "Base")); @@ -264,7 +265,7 @@ TEST_F(PythonParserTest, ClassInheritance) EXPECT_EQ(pyname.parent, derived.id); pyname = queryFile("classes.py", - (odb::query::line_start == 21 && + (odb::query::line_start == 23 && odb::query::column_start == 16 && odb::query::value == "Derived")); @@ -272,7 +273,7 @@ TEST_F(PythonParserTest, ClassInheritance) EXPECT_EQ(pyname.parent, derived2.id); pyname = queryFile("classes.py", - (odb::query::line_start == 21 && + (odb::query::line_start == 23 && odb::query::column_start == 25 && odb::query::value == "Base")); @@ -301,7 +302,7 @@ TEST_F(PythonParserTest, ClassMethod) EXPECT_EQ(bar.parent, base.id); model::PYName test = queryFile("classes.py", - (odb::query::line_start == 12 && + (odb::query::line_start == 14 && odb::query::is_definition == true && odb::query::type == "function")); @@ -337,7 +338,7 @@ TEST_F(PythonParserTest, LocalVariable) odb::query::value == "def local_var()"); pyname = queryFile("functions.py", - (odb::query::line_start == 96 && + (odb::query::line_start == 97 && odb::query::value == "a = 2")); EXPECT_EQ(pyname.type, "statement"); @@ -345,10 +346,33 @@ TEST_F(PythonParserTest, LocalVariable) EXPECT_EQ(pyname.parent, func.id); pyname = queryFile("functions.py", - (odb::query::line_start == 97 && + (odb::query::line_start == 98 && odb::query::value == "for i in range(0,10):\n")); EXPECT_EQ(pyname.type, "statement"); EXPECT_EQ(pyname.is_definition, true); EXPECT_EQ(pyname.parent, func.id); } + +TEST_F(PythonParserTest, ImportModule) +{ + model::PYName pyname; + + pyname = queryFile("imports.py", + (odb::query::line_start == 1 && + odb::query::value == "import classes")); + + EXPECT_EQ(pyname.is_import, true); + + pyname = queryFile("imports.py", + (odb::query::line_start == 2 && + odb::query::value == "import os")); + + EXPECT_EQ(pyname.is_import, true); + + pyname = queryFile("imports.py", + (odb::query::line_start == 3 && + odb::query::value == "from functions import mul")); + + EXPECT_EQ(pyname.is_import, true); +} From 8cbea736e6d972116753752ea668fa831907ebca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Wed, 20 Nov 2024 15:56:05 +0100 Subject: [PATCH 142/156] [PythonTest] Parser test - builtin variable --- plugins/python/test/src/pythonparsertest.cpp | 49 ++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp index c3d6bafcf..007282b24 100644 --- a/plugins/python/test/src/pythonparsertest.cpp +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -60,6 +60,14 @@ TEST_F(PythonParserTest, FilesAreInDatabase) file = _db->query_value(odb::query::filename == "functions.py"); EXPECT_EQ(file.type, "PY"); EXPECT_EQ(file.parseStatus, model::File::PSFullyParsed); + + file = _db->query_value(odb::query::filename == "classes.py"); + EXPECT_EQ(file.type, "PY"); + EXPECT_EQ(file.parseStatus, model::File::PSFullyParsed); + + file = _db->query_value(odb::query::filename == "imports.py"); + EXPECT_EQ(file.type, "PY"); + EXPECT_EQ(file.parseStatus, model::File::PSFullyParsed); }); } @@ -376,3 +384,44 @@ TEST_F(PythonParserTest, ImportModule) EXPECT_EQ(pyname.is_import, true); } + +TEST_F(PythonParserTest, BuiltinVariable) +{ + model::PYName pyname; + + pyname = queryFile("imports.py", + (odb::query::line_start == 2 && + odb::query::value == "import os")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("imports.py", + (odb::query::line_start == 6 && + odb::query::value == "print")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("imports.py", + (odb::query::line_start == 12 && + odb::query::value == "getpid")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("functions.py", + (odb::query::line_start == 85 && + odb::query::value == "str")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("functions.py", + (odb::query::line_start == 85 && + odb::query::value == "List")); + + EXPECT_EQ(pyname.is_builtin, true); + + pyname = queryFile("functions.py", + (odb::query::line_start == 98 && + odb::query::value == "range")); + + EXPECT_EQ(pyname.is_builtin, true); +} From 4be4e94e2b1de928cd2d5981e9efc6805d46d565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 23 Nov 2024 12:15:16 +0100 Subject: [PATCH 143/156] [PythonTest] Parser test - ClassType, ReferenceID --- plugins/python/test/src/pythonparsertest.cpp | 62 ++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/plugins/python/test/src/pythonparsertest.cpp b/plugins/python/test/src/pythonparsertest.cpp index 007282b24..c656e4b29 100644 --- a/plugins/python/test/src/pythonparsertest.cpp +++ b/plugins/python/test/src/pythonparsertest.cpp @@ -252,6 +252,29 @@ TEST_F(PythonParserTest, FunctionCall) EXPECT_EQ(pyname.is_call, false); } +TEST_F(PythonParserTest, ClassType) +{ + model::PYName pyname; + + pyname = queryFile("classes.py", + (odb::query::line_start == 1 && + odb::query::value == "class Base:\n")); + + EXPECT_EQ(pyname.type, "class"); + + pyname = queryFile("classes.py", + (odb::query::line_start == 19 && + odb::query::value == "class Derived(Base):\n")); + + EXPECT_EQ(pyname.type, "class"); + + pyname = queryFile("classes.py", + (odb::query::line_start == 23 && + odb::query::value == "class Derived2(Derived, Base):\n")); + + EXPECT_EQ(pyname.type, "class"); +} + TEST_F(PythonParserTest, ClassInheritance) { model::PYName pyname; @@ -425,3 +448,42 @@ TEST_F(PythonParserTest, BuiltinVariable) EXPECT_EQ(pyname.is_builtin, true); } + +TEST_F(PythonParserTest, ReferenceID) +{ + model::PYName pyname; + + model::PYName func = queryFile("functions.py", odb::query::value == "def mul(a, b)"); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 11); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 14); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 17); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 22); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 26); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 27); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 43); + EXPECT_EQ(pyname.ref_id, func.id); + + pyname = queryFile("functions.py", odb::query::value == "mul" && + odb::query::line_start == 44); + EXPECT_EQ(pyname.ref_id, func.id); +} From f42f59e77f96db35e4c95bbdc20f66f83a426eb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 23 Nov 2024 16:07:56 +0100 Subject: [PATCH 144/156] [PythonTest] Service test - node properties, reference tests --- plugins/python/test/sources/classes.py | 6 + plugins/python/test/src/pythonservicetest.cpp | 305 +++++++++++++++++- 2 files changed, 300 insertions(+), 11 deletions(-) diff --git a/plugins/python/test/sources/classes.py b/plugins/python/test/sources/classes.py index b1cf4474a..888315f33 100644 --- a/plugins/python/test/sources/classes.py +++ b/plugins/python/test/sources/classes.py @@ -23,3 +23,9 @@ def __init__(self) -> None: class Derived2(Derived, Base): def __init__(self) -> None: pass + +base = Base() + +def func(): + class A: + z = 2 diff --git a/plugins/python/test/src/pythonservicetest.cpp b/plugins/python/test/src/pythonservicetest.cpp index 2af3ea187..7f3446870 100644 --- a/plugins/python/test/src/pythonservicetest.cpp +++ b/plugins/python/test/src/pythonservicetest.cpp @@ -5,6 +5,7 @@ #include #include +#include using namespace cc; using namespace cc::service::language; @@ -22,32 +23,60 @@ class PythonServiceTest : public ::testing::Test webserver::ServerContext(std::string(), boost::program_options::variables_map())) ) { - } - - model::File queryFile(const std::string& filename) - { - return _transaction([&, this]() { - return _db->query_value(odb::query::filename == filename); - }); + loadFile("functions.py"); + loadFile("classes.py"); + loadFile("imports.py"); } AstNodeInfo getAstNodeInfoByPosition(const std::string& filename, int32_t line, int32_t column) { - model::File file = queryFile(filename); + AstNodeInfo nodeInfo; + + if (m_files.count(filename) == 0) + { + return nodeInfo; + } + + model::FileId file_id = m_files[filename]; service::core::FilePosition filePos; - filePos.file = std::to_string(file.id); + filePos.file = std::to_string(file_id); filePos.pos.line = line; filePos.pos.column = column; - AstNodeInfo nodeInfo; _pythonservice->getAstNodeInfoByPosition(nodeInfo, filePos); return nodeInfo; } + size_t referenceFinder(const std::vector& references, + const std::vector& positions) + { + size_t c = 0; + for (const AstNodeInfo& info : references) + { + size_t line = info.range.range.startpos.line; + size_t column = info.range.range.startpos.column; + + for (const model::Position& pos : positions) + { + if (pos.line == line && pos.column == column) c++; + } + } + + return c; + } + private: std::shared_ptr _db; util::OdbTransaction _transaction; + std::unordered_map m_files; + void loadFile(const std::string& filename) + { + _transaction([&, this]() { + model::File file = _db->query_value(odb::query::filename == filename); + m_files.emplace(filename, file.id); + }); + } protected: std::shared_ptr _pythonservice; @@ -55,9 +84,263 @@ class PythonServiceTest : public ::testing::Test TEST_F(PythonServiceTest, AstNodeInfoByPosition) { + AstNodeInfo nodeInfo; + // Simulating click on line 1 column 5 - AstNodeInfo nodeInfo = getAstNodeInfoByPosition("functions.py", 1, 5); + nodeInfo = getAstNodeInfoByPosition("functions.py", 1, 5); EXPECT_EQ(nodeInfo.astNodeValue, "def hello_world()"); EXPECT_EQ(nodeInfo.symbolType, "function"); + + nodeInfo = getAstNodeInfoByPosition("functions.py", 14, 12); + + EXPECT_EQ(nodeInfo.astNodeValue, "mul"); + EXPECT_EQ(nodeInfo.symbolType, "statement"); + + nodeInfo = getAstNodeInfoByPosition("functions.py", 24, 10); + + EXPECT_EQ(nodeInfo.astNodeValue, "MyLib"); + EXPECT_EQ(nodeInfo.symbolType, "statement"); +} + +TEST_F(PythonServiceTest, NodeProperties) +{ + AstNodeInfo nodeInfo; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 7, 5); + std::map map; + + _pythonservice->getProperties(map, nodeInfo.id); + EXPECT_EQ(nodeInfo.astNodeValue, "def mul(a, b)"); + + EXPECT_EQ(map.count("Full name"), 1); + EXPECT_EQ(map["Full name"], "functions.mul"); + EXPECT_EQ(map.count("Builtin"), 1); + EXPECT_EQ(map["Builtin"], "false"); + EXPECT_EQ(map.count("Function call"), 1); + EXPECT_EQ(map["Function call"], "false"); +} + +TEST_F(PythonServiceTest, NodePropertiesBuiltinCall) +{ + AstNodeInfo nodeInfo; + + nodeInfo = getAstNodeInfoByPosition("imports.py", 12, 17); + EXPECT_EQ(nodeInfo.astNodeValue, "getpid"); + + std::map map; + + _pythonservice->getProperties(map, nodeInfo.id); + EXPECT_EQ(map.count("Full name"), 1); + EXPECT_EQ(map["Full name"], "imports.getpid"); + EXPECT_EQ(map.count("Builtin"), 1); + EXPECT_EQ(map["Builtin"], "true"); + EXPECT_EQ(map.count("Function call"), 1); + EXPECT_EQ(map["Function call"], "true"); +} + +TEST_F(PythonServiceTest, NodeDefinition) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 11, 12); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::DEFINITION, {}); + + EXPECT_EQ(references.size(), 1); + EXPECT_EQ(references[0].symbolType, "function"); + EXPECT_EQ(references[0].astNodeValue, "def mul(a, b)"); + + references = {}; + nodeInfo = getAstNodeInfoByPosition("functions.py", 29, 6); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::DEFINITION, {}); + + EXPECT_EQ(references.size(), 1); + EXPECT_EQ(references[0].symbolType, "statement"); + EXPECT_EQ(references[0].astNodeValue, "mylib = {\n"); + + references = {}; + nodeInfo = getAstNodeInfoByPosition("classes.py", 27, 8); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::DEFINITION, {}); + + EXPECT_EQ(references.size(), 1); + EXPECT_EQ(references[0].symbolType, "class"); + EXPECT_EQ(references[0].astNodeValue, "class Base:\n"); +} + +TEST_F(PythonServiceTest, NodeUsage) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 16, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::USAGE, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "mylib = {\n"); + EXPECT_EQ(references.size(), 2); + + size_t found = referenceFinder(references, {{29, 6}, {46, 11}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeDataMembers) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("classes.py", 1, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::DATA_MEMBER, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "class Base:\n"); + EXPECT_EQ(references.size(), 2); + + size_t found = referenceFinder(references, {{2, 5}, {3, 5}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeLocalVariables) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 96, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::LOCAL_VAR, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def local_var()"); + EXPECT_EQ(references.size(), 3); + + size_t found = referenceFinder(references, {{97, 5}, {98, 5}, {99, 9}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeParent) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 97, 5); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::PARENT, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "a = 2"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{96, 1}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeParentFunction) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("classes.py", 31, 9); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::PARENT_FUNCTION, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "z = 2"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{29, 1}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeCaller) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 13, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::CALLER, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def mul3()"); + EXPECT_EQ(references.size(), 2); + + size_t found = referenceFinder(references, {{28, 6}, {45, 11}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeThisCalls) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 10, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::THIS_CALLS, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def mul2(a, b)"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{11, 12}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeMethod) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("classes.py", 1, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::METHOD, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "class Base:\n"); + EXPECT_EQ(references.size(), 3); + + size_t found = referenceFinder(references, {{5, 5}, {8, 5}, {11, 5}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeBaseClass) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("classes.py", 19, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::BASE_CLASS, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "class Derived(Base):\n"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{1, 1}}); + EXPECT_EQ(found, references.size()); + + // ---------- + + references = {}; + nodeInfo = getAstNodeInfoByPosition("classes.py", 23, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::BASE_CLASS, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "class Derived2(Derived, Base):\n"); + EXPECT_EQ(references.size(), 2); + + found = referenceFinder(references, {{1, 1}, {19, 1}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeFunctionParam) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 4, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::PARAMETER, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def runner(func, param1, param2)"); + EXPECT_EQ(references.size(), 3); + + size_t found = referenceFinder(references, {{4, 12}, {4, 18}, {4, 26}}); + EXPECT_EQ(found, references.size()); +} + +TEST_F(PythonServiceTest, NodeAnnotation) +{ + AstNodeInfo nodeInfo; + std::vector references; + + nodeInfo = getAstNodeInfoByPosition("functions.py", 69, 1); + _pythonservice->getReferences(references, nodeInfo.id, PythonServiceHandler::ANNOTATION, {}); + + EXPECT_EQ(nodeInfo.astNodeValue, "def annotation(a, b) -> None"); + EXPECT_EQ(references.size(), 1); + + size_t found = referenceFinder(references, {{69, 24}}); + EXPECT_EQ(found, references.size()); } From bd6169e2cf9394a77cc12ae655dff548b4c21d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 23 Nov 2024 17:58:49 +0100 Subject: [PATCH 145/156] pycache gitignore --- plugins/python/test/sources/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 plugins/python/test/sources/.gitignore diff --git a/plugins/python/test/sources/.gitignore b/plugins/python/test/sources/.gitignore new file mode 100644 index 000000000..bee8a64b7 --- /dev/null +++ b/plugins/python/test/sources/.gitignore @@ -0,0 +1 @@ +__pycache__ From b0b111d90c3b39a1a8772084a60712a6ed266ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 23 Nov 2024 17:59:22 +0100 Subject: [PATCH 146/156] [PythonService] move boolToString to util --- plugins/python/service/include/service/pythonservice.h | 1 - plugins/python/service/src/pythonservice.cpp | 7 ++++--- util/include/util/util.h | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index a0443da89..13ab53c00 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -240,7 +240,6 @@ class PythonServiceHandler : virtual public LanguageServiceIf std::shared_ptr _datadir; const cc::webserver::ServerContext& _context; - inline const char* boolToString(bool b) { return b ? "true" : "false"; } void setInfoProperties(AstNodeInfo& info, const model::PYName& pyname); }; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index e30ffab25..72e33d6a3 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include "diagram.h" @@ -89,19 +90,19 @@ void PythonServiceHandler::getProperties( return_.emplace("Full name", pyname.full_name); } - return_.emplace("Builtin", PythonServiceHandler::boolToString(pyname.is_builtin)); + return_.emplace("Builtin", util::boolToString(pyname.is_builtin)); if(!pyname.type_hint.empty()) { return_.emplace("Type hint", pyname.type_hint); } - return_.emplace("Function call", PythonServiceHandler::boolToString(pyname.is_call)); + return_.emplace("Function call", util::boolToString(pyname.is_call)); #ifndef NDEBUG return_.emplace("ID", std::to_string(pyname.id)); return_.emplace("REF_ID", std::to_string(pyname.ref_id)); - return_.emplace("DEFINITION", PythonServiceHandler::boolToString(pyname.is_definition)); + return_.emplace("DEFINITION", util::boolToString(pyname.is_definition)); #endif return; diff --git a/util/include/util/util.h b/util/include/util/util.h index 1af6f90d9..7f8d78b37 100644 --- a/util/include/util/util.h +++ b/util/include/util/util.h @@ -29,6 +29,10 @@ std::string textRange( */ std::string escapeHtml(const std::string& str_); +inline const char* boolToString(bool b) { + return b ? "true" : "false"; +} + } } From e322c2beee649d7105675e37a7e7b4328e9b2b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 23 Nov 2024 18:34:36 +0100 Subject: [PATCH 147/156] [PythonService] Cleanup --- .../service/include/service/pythonservice.h | 107 +++------------- plugins/python/service/src/pythonservice.cpp | 121 ++++++------------ 2 files changed, 58 insertions(+), 170 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 13ab53c00..5e6c231ad 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -49,7 +49,7 @@ class PythonServiceHandler : virtual public LanguageServiceIf void getDocumentation( std::string& return_, - const core::AstNodeId& astNodeId_) override; + const core::AstNodeId& astNodeId_) override {}; void getProperties( std::map& return_, @@ -100,121 +100,46 @@ class PythonServiceHandler : virtual public LanguageServiceIf const core::AstNodeId& astNodeId_, const std::int32_t referenceId_, const core::FileId& fileId_, - const std::vector& tags_) override; + const std::vector& tags_) override {}; void getReferencesPage( std::vector& return_, const core::AstNodeId& astNodeId_, const std::int32_t referenceId_, const std::int32_t pageSize_, - const std::int32_t pageNo_) override; + const std::int32_t pageNo_) override {}; void getFileReferenceTypes( std::map& return_, - const core::FileId& fileId_) override; + const core::FileId& fileId_) override {}; void getFileReferences( std::vector& return_, const core::FileId& fileId_, - const std::int32_t referenceId_) override; + const std::int32_t referenceId_) override {}; std::int32_t getFileReferenceCount( const core::FileId& fileId_, - const std::int32_t referenceId_) override; + const std::int32_t referenceId_) override {}; void getSyntaxHighlight( std::vector& return_, - const core::FileRange& range_) override; + const core::FileRange& range_) override {}; enum ReferenceType { - DEFINITION, /*!< By this option the definition(s) of the AST node can be - queried. However according to the "one definition rule" a named entity - can have only one definition, in a parsing several definitions might be - available. This is the case when the project is built for several targets - and in the different builds different definitions are defined for an - entity (e.g. because of an #ifdef section). */ - - DECLARATION, /*!< By this options the declaration(s) of the AST node can be - queried. */ - - USAGE, /*!< By this option the usages of the AST node can be queried, i.e. - the nodes of which the entity hash is identical to the queried one. */ - - THIS_CALLS, /*!< Get function calls in a function. WARNING: If the - definition of the AST node is not unique then it returns the callees of - one of them. */ - - CALLS_OF_THIS, /*!< Get calls of a function. */ - - CALLEE, /*!< Get called functions definitions. WARNING: If the definition of - the AST node is not unique then it returns the callees of one of them. */ - - CALLER, /*!< Get caller functions. */ - - VIRTUAL_CALL, /*!< A function may be used virtually on a base type object. - The exact type of the object is based on dynamic information, which can't - be determined statically. Weak usage returns these possible calls. */ - - FUNC_PTR_CALL, /*!< Functions can be assigned to function pointers which - can be invoked later. This option returns these invocations. */ - - PARAMETER, /*!< This option returns the parameters of a function. */ - - LOCAL_VAR, /*!< This option returns the local variables of a function. */ - - RETURN_TYPE, /*!< This option returns the return type of a function. */ - - OVERRIDE, /*!< This option returns the functions which the given function - overrides. */ - - OVERRIDDEN_BY, /*!< This option returns the overrides of a function. */ - - READ, /*!< This option returns the places where a variable is read. */ - - WRITE, /*!< This option returns the places where a variable is written. */ - - TYPE, /*!< This option returns the type of a variable. */ - - ALIAS, /*!< Types may have aliases, e.g. by typedefs. */ - - INHERIT_FROM, /*!< Types from which the queried type inherits. */ - - INHERIT_BY, /*!< Types by which the queried type is inherited. */ - - DATA_MEMBER, /*!< Data members of a class. */ - - METHOD, /*!< Members of a class. */ - - FRIEND, /*!< The friends of a class. */ - - UNDERLYING_TYPE, /*!< Underlying type of a typedef. */ - - ENUM_CONSTANTS, /*!< Enum constants. */ - - EXPANSION, /*!< Macro expansion. */ - - UNDEFINITION, /*!< Macro undefinition. */ - + DEFINITION, + USAGE, + THIS_CALLS, + CALLER, + PARAMETER, + LOCAL_VAR, + DATA_MEMBER, + METHOD, PARENT, - PARENT_FUNCTION, - ANNOTATION, - - BASE_CLASS, - }; - - enum FileReferenceType - { - INCLUDES, /*!< Included source files in the current source file after the - inclusion directive. */ - - TYPES, /*!< User defined data types such as classes, structs etc. */ - - FUNCTIONS, /*!< Functions in the current source file. */ - - MACROS, /*!< Macros in the current source file. */ + BASE_CLASS }; enum DiagramType diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 72e33d6a3..57058c7ab 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -21,6 +21,9 @@ PythonServiceHandler::PythonServiceHandler( void PythonServiceHandler::getFileTypes( std::vector& return_) { +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif return_.push_back("PY"); return_.push_back("Dir"); return; @@ -30,7 +33,9 @@ void PythonServiceHandler::getAstNodeInfo( AstNodeInfo& return_, const core::AstNodeId& astNodeId_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); PythonServiceHandler::setInfoProperties(return_, pyname); @@ -41,7 +46,9 @@ void PythonServiceHandler::getAstNodeInfoByPosition( AstNodeInfo& return_, const core::FilePosition& fpos_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif model::PYName node = PythonServiceHandler::queryNodeByPosition(fpos_); PythonServiceHandler::setInfoProperties(return_, node); return; @@ -51,7 +58,9 @@ void PythonServiceHandler::getSourceText( std::string& return_, const core::AstNodeId& astNodeId_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); core::ProjectServiceHandler projectService(_db, _datadir, _context); @@ -70,19 +79,11 @@ void PythonServiceHandler::getSourceText( return; } -void PythonServiceHandler::getDocumentation( - std::string& return_, - const core::AstNodeId& astNodeId_) -{ - LOG(info) << "[PYSERVICE] " << __func__; - return; -} - void PythonServiceHandler::getProperties( std::map& return_, const core::AstNodeId& astNodeId_) { - LOG(info) << "[PYSERVICE] " << __func__; + LOG(info) << "[PYTHONSERVICE]" << __func__; model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); if(!pyname.full_name.empty()) @@ -112,7 +113,9 @@ void PythonServiceHandler::getDiagramTypes( std::map& return_, const core::AstNodeId& astNodeId_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif const model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); if (pyname.is_import == true) return; @@ -144,7 +147,9 @@ void PythonServiceHandler::getDiagram( const core::AstNodeId& astNodeId_, const std::int32_t diagramId_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif python::Diagram diagram(_db, _datadir, _context); model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); @@ -179,7 +184,9 @@ void PythonServiceHandler::getDiagramLegend( std::string& return_, const std::int32_t diagramId_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif python::Diagram diagram(_db, _datadir, _context); util::Graph graph = [&]() @@ -206,7 +213,9 @@ void PythonServiceHandler::getFileDiagramTypes( std::map& return_, const core::FileId& fileId_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif return_.emplace("Module dependency", MODULE_DEPENDENCY); return; } @@ -216,7 +225,9 @@ void PythonServiceHandler::getFileDiagram( const core::FileId& fileId_, const int32_t diagramId_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif python::Diagram diagram(_db, _datadir, _context); util::Graph graph = [&]() @@ -240,7 +251,9 @@ void PythonServiceHandler::getFileDiagramLegend( std::string& return_, const std::int32_t diagramId_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif python::Diagram diagram(_db, _datadir, _context); util::Graph graph = [&]() @@ -264,7 +277,9 @@ void PythonServiceHandler::getReferenceTypes( std::map& return_, const core::AstNodeId& astNodeId) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; +#endif return_.emplace("Definition", DEFINITION); return_.emplace("Usage", USAGE); return_.emplace("Parent", PARENT); @@ -302,9 +317,10 @@ void PythonServiceHandler::getReferences( const std::int32_t referenceId_, const std::vector& tags_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; LOG(info) << "astNodeID: " << astNodeId_; - +#endif std::vector nodes = PythonServiceHandler::queryReferences(astNodeId_, referenceId_); for(const model::PYName& pyname : nodes) @@ -321,67 +337,14 @@ std::int32_t PythonServiceHandler::getReferenceCount( const core::AstNodeId& astNodeId_, const std::int32_t referenceId_) { - LOG(info) << "[PYSERVICE] " << __func__; +#ifndef NDEBUG + LOG(info) << "[PYTHONSERVICE]" << __func__; LOG(info) << "astNodeID: " << astNodeId_; +#endif return PythonServiceHandler::queryReferences(astNodeId_, referenceId_).size(); } -void PythonServiceHandler::getReferencesInFile( - std::vector& return_, - const core::AstNodeId& astNodeId_, - const std::int32_t referenceId_, - const core::FileId& fileId_, - const std::vector& tags_) -{ - LOG(info) << "[PYSERVICE] " << __func__; - return; -} - -void PythonServiceHandler::getReferencesPage( - std::vector& return_, - const core::AstNodeId& astNodeId_, - const std::int32_t referenceId_, - const std::int32_t pageSize_, - const std::int32_t pageNo_) -{ - LOG(info) << "[PYSERVICE] " << __func__; - return; -} - -void PythonServiceHandler::getFileReferenceTypes( - std::map& return_, - const core::FileId& fileId_) -{ - LOG(info) << "[PYSERVICE] " << __func__; - return; -} - -void PythonServiceHandler::getFileReferences( - std::vector& return_, - const core::FileId& fileId_, - const std::int32_t referenceId_) -{ - LOG(info) << "[PYSERVICE] " << __func__; - return; -} - -std::int32_t PythonServiceHandler::getFileReferenceCount( - const core::FileId& fileId_, - const std::int32_t referenceId_) -{ - LOG(info) << "[PYSERVICE] " << __func__; - return 0; -} - -void PythonServiceHandler::getSyntaxHighlight( - std::vector& return_, - const core::FileRange& range_) -{ - LOG(info) << "[PYSERVICE] " << __func__; - return; -} - model::PYName PythonServiceHandler::queryNodeByID(const std::string& id) { return _transaction([&]() @@ -393,7 +356,7 @@ model::PYName PythonServiceHandler::queryNodeByID(const std::string& id) { pyname = *nodes.begin(); }else{ - LOG(info) << "[PYSERVICE] Node not found! (id = " << id << ")"; + LOG(info) << "[PYTHONSERVICE] Node not found! (id = " << id << ")"; core::InvalidId ex; ex.__set_msg("Node not found!"); throw ex; @@ -427,7 +390,7 @@ model::PYName PythonServiceHandler::queryNodeByPosition(const core::FilePosition } } }else{ - LOG(info) << "[PYSERVICE] Node not found! (line = " << fpos.pos.line << " column = " << fpos.pos.column << ")"; + LOG(info) << "[PYTHONSERVICE] Node not found! (line = " << fpos.pos.line << " column = " << fpos.pos.column << ")"; core::InvalidInput ex; ex.__set_msg("Node not found!"); throw ex; From df85f871919ef4ed05c367ffae02df13cdaf7dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 23 Nov 2024 19:28:10 +0100 Subject: [PATCH 148/156] [PythonService] Make PythonDiagram friend class --- .../service/include/service/pythonservice.h | 16 ++++----- plugins/python/service/src/diagram.cpp | 35 +++++++++---------- plugins/python/service/src/diagram.h | 7 ++-- plugins/python/service/src/pythonservice.cpp | 8 ++--- 4 files changed, 30 insertions(+), 36 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 5e6c231ad..ae6662e66 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -27,6 +27,7 @@ namespace language class PythonServiceHandler : virtual public LanguageServiceIf { + friend class PythonDiagram; public: PythonServiceHandler( std::shared_ptr db_, @@ -151,14 +152,6 @@ class PythonServiceHandler : virtual public LanguageServiceIf CLASS_OVERVIEW }; - model::PYName queryNodeByID(const std::string& id); - model::PYName queryNodeByPosition(const core::FilePosition& fpos); - std::vector queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); - std::vector queryNodesInFile(const core::FileId& fileId, bool definitions); - std::vector queryNodes(const odb::query& odb_query); - std::vector transformReferences(const std::vector& references, const model::PYNameID& id); - std::string getNodeLineValue(const model::PYName& pyname); - private: std::shared_ptr _db; util::OdbTransaction _transaction; @@ -166,6 +159,13 @@ class PythonServiceHandler : virtual public LanguageServiceIf const cc::webserver::ServerContext& _context; void setInfoProperties(AstNodeInfo& info, const model::PYName& pyname); + model::PYName queryNodeByID(const std::string& id); + model::PYName queryNodeByPosition(const core::FilePosition& fpos); + std::vector queryReferences(const core::AstNodeId& astNodeId, const std::int32_t referenceId); + std::vector queryNodesInFile(const core::FileId& fileId, bool definitions); + std::vector queryNodes(const odb::query& odb_query); + std::vector transformReferences(const std::vector& references, const model::PYNameID& id); + std::string getNodeLineValue(const model::PYName& pyname); }; } // language diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index e598a7ee1..5be893a0d 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -6,16 +6,14 @@ namespace service { namespace language { -namespace python -{ -Diagram::Diagram( +PythonDiagram::PythonDiagram( std::shared_ptr db_, std::shared_ptr datadir_, const cc::webserver::ServerContext& context_) : m_pythonService(db_, datadir_, context_), m_projectService(db_, datadir_, context_){} -util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) +util::Graph PythonDiagram::getFunctionCallDiagram(const model::PYName& pyname) { // Query calls const std::vector this_calls = m_pythonService.queryReferences(std::to_string(pyname.id), PythonServiceHandler::THIS_CALLS); @@ -73,7 +71,7 @@ util::Graph Diagram::getFunctionCallDiagram(const model::PYName& pyname) return graph; } -util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) +util::Graph PythonDiagram::getModuleDiagram(const core::FileId& fileId) { util::Graph graph; graph.setAttribute("rankdir", "LR"); @@ -140,7 +138,7 @@ util::Graph Diagram::getModuleDiagram(const core::FileId& fileId) return graph; } -util::Graph Diagram::getUsageDiagram(const model::PYName& pyname) +util::Graph PythonDiagram::getUsageDiagram(const model::PYName& pyname) { util::Graph graph; graph.setAttribute("rankdir", "LR"); @@ -161,7 +159,7 @@ util::Graph Diagram::getUsageDiagram(const model::PYName& pyname) return graph; } -void Diagram::addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType) +void PythonDiagram::addFunctionNode(util::Graph& graph_, const util::Graph::Node& centerNode, const model::PYName& pyname, const NodeType& nodeType) { util::Graph::Node node = addPYNameNode(graph_, pyname, true); decorateNode(graph_, node, nodeType); @@ -176,7 +174,7 @@ void Diagram::addFunctionNode(util::Graph& graph_, const util::Graph::Node& cent } } -util::Graph::Subgraph Diagram::getFileSubgraph(util::Graph& graph_, const model::PYName& pyname) +util::Graph::Subgraph PythonDiagram::getFileSubgraph(util::Graph& graph_, const model::PYName& pyname) { const core::FileId fileId = std::to_string(pyname.file_id); auto it = m_subgraphs.find(fileId); @@ -207,7 +205,7 @@ util::Graph::Subgraph Diagram::getFileSubgraph(util::Graph& graph_, const model: return subgraph; } -util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph) +util::Graph::Node PythonDiagram::addPYNameNode(util::Graph& graph_, const model::PYName& pyname, bool addSubgraph) { const util::Graph::Subgraph subgraph = (addSubgraph) ? getFileSubgraph(graph_, pyname) : util::Graph::Subgraph(); @@ -225,7 +223,7 @@ util::Graph::Node Diagram::addPYNameNode(util::Graph& graph_, const model::PYNam return node; } -util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo) +util::Graph::Node PythonDiagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo) { util::Graph::Node node = graph_.getOrCreateNode("f" + fileInfo.id); graph_.setNodeAttribute(node, "label", fileInfo.path); @@ -233,7 +231,7 @@ util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo return node; } -util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType) +util::Graph::Node PythonDiagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo, const util::Graph::Node& centerNode, const NodeType& nodeType) { /* We might need to add a file path multiple times to the diagram. Since we need unique ids, we will differentiate nodes based on starting character: @@ -256,7 +254,7 @@ util::Graph::Node Diagram::addFileNode(util::Graph& graph_, const core::FileInfo return node; } -void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeType& nodeType) +void PythonDiagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const NodeType& nodeType) { graph_.setNodeAttribute(node_, "style", "filled"); @@ -305,7 +303,7 @@ void Diagram::decorateNode(util::Graph& graph_, util::Graph::Node& node_, const } } -util::Graph Diagram::getFunctionCallDiagramLegend() +util::Graph PythonDiagram::getFunctionCallDiagramLegend() { util::Graph graph; graph.setAttribute("rankdir", "LR"); @@ -321,7 +319,7 @@ util::Graph Diagram::getFunctionCallDiagramLegend() return graph; } -util::Graph Diagram::getUsageDiagramLegend() +util::Graph PythonDiagram::getUsageDiagramLegend() { util::Graph graph; graph.setAttribute("rankdir", "LR"); @@ -335,7 +333,7 @@ util::Graph Diagram::getUsageDiagramLegend() return graph; } -util::Graph Diagram::getModuleDiagramLegend() +util::Graph PythonDiagram::getModuleDiagramLegend() { util::Graph graph; graph.setAttribute("rankdir", "LR"); @@ -349,7 +347,7 @@ util::Graph Diagram::getModuleDiagramLegend() return graph; } -void Diagram::addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text, bool shape) +void PythonDiagram::addLegendNode(util::Graph& graph_, const NodeType& nodeType, const std::string& text, bool shape) { util::Graph::Node node = graph_.createNode(); graph_.setNodeAttribute(node, "label", ""); @@ -368,7 +366,7 @@ void Diagram::addLegendNode(util::Graph& graph_, const NodeType& nodeType, const graph_.setNodeAttribute(explanation, "label", text); } -util::Graph Diagram::getClassDiagram(const model::PYName& pyname) +util::Graph PythonDiagram::getClassDiagram(const model::PYName& pyname) { util::Graph graph; graph.setAttribute("rankdir", "BT"); @@ -400,7 +398,7 @@ util::Graph Diagram::getClassDiagram(const model::PYName& pyname) return graph; } -std::string Diagram::getClassTable(const model::PYName& pyname) +std::string PythonDiagram::getClassTable(const model::PYName& pyname) { auto getVisibility = [](const std::string& str) { @@ -496,7 +494,6 @@ std::string Diagram::getClassTable(const model::PYName& pyname) label += "
"; return label; } -} // python } // language } // service } // cc diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index d2c1711c9..8083cf326 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -17,11 +17,9 @@ namespace service { namespace language { -namespace python -{ - class Diagram { + class PythonDiagram { public: - Diagram( + PythonDiagram( std::shared_ptr db_, std::shared_ptr datadir_, const cc::webserver::ServerContext& context_); @@ -60,7 +58,6 @@ namespace python util::Graph::Subgraph getFileSubgraph(util::Graph& graph_, const model::PYName& pyname); std::string getClassTable(const model::PYName& pyname); }; -} // python } // language } // service } // cc diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 57058c7ab..98aa7578f 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -150,7 +150,7 @@ void PythonServiceHandler::getDiagram( #ifndef NDEBUG LOG(info) << "[PYTHONSERVICE]" << __func__; #endif - python::Diagram diagram(_db, _datadir, _context); + PythonDiagram diagram(_db, _datadir, _context); model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); util::Graph graph = [&]() @@ -187,7 +187,7 @@ void PythonServiceHandler::getDiagramLegend( #ifndef NDEBUG LOG(info) << "[PYTHONSERVICE]" << __func__; #endif - python::Diagram diagram(_db, _datadir, _context); + PythonDiagram diagram(_db, _datadir, _context); util::Graph graph = [&]() { @@ -228,7 +228,7 @@ void PythonServiceHandler::getFileDiagram( #ifndef NDEBUG LOG(info) << "[PYTHONSERVICE]" << __func__; #endif - python::Diagram diagram(_db, _datadir, _context); + PythonDiagram diagram(_db, _datadir, _context); util::Graph graph = [&]() { @@ -254,7 +254,7 @@ void PythonServiceHandler::getFileDiagramLegend( #ifndef NDEBUG LOG(info) << "[PYTHONSERVICE]" << __func__; #endif - python::Diagram diagram(_db, _datadir, _context); + PythonDiagram diagram(_db, _datadir, _context); util::Graph graph = [&]() { From 62c93caf75d6eb9c4bcdd1f6a0ae8e13b77c43b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sat, 23 Nov 2024 20:56:48 +0100 Subject: [PATCH 149/156] [PythonService] Debug message spacing --- plugins/python/service/src/pythonservice.cpp | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 98aa7578f..81307d8df 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -22,7 +22,7 @@ void PythonServiceHandler::getFileTypes( std::vector& return_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif return_.push_back("PY"); return_.push_back("Dir"); @@ -34,7 +34,7 @@ void PythonServiceHandler::getAstNodeInfo( const core::AstNodeId& astNodeId_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); @@ -47,7 +47,7 @@ void PythonServiceHandler::getAstNodeInfoByPosition( const core::FilePosition& fpos_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif model::PYName node = PythonServiceHandler::queryNodeByPosition(fpos_); PythonServiceHandler::setInfoProperties(return_, node); @@ -59,7 +59,7 @@ void PythonServiceHandler::getSourceText( const core::AstNodeId& astNodeId_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); @@ -83,7 +83,7 @@ void PythonServiceHandler::getProperties( std::map& return_, const core::AstNodeId& astNodeId_) { - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); if(!pyname.full_name.empty()) @@ -114,7 +114,7 @@ void PythonServiceHandler::getDiagramTypes( const core::AstNodeId& astNodeId_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif const model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); @@ -148,7 +148,7 @@ void PythonServiceHandler::getDiagram( const std::int32_t diagramId_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif PythonDiagram diagram(_db, _datadir, _context); @@ -185,7 +185,7 @@ void PythonServiceHandler::getDiagramLegend( const std::int32_t diagramId_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif PythonDiagram diagram(_db, _datadir, _context); @@ -214,7 +214,7 @@ void PythonServiceHandler::getFileDiagramTypes( const core::FileId& fileId_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif return_.emplace("Module dependency", MODULE_DEPENDENCY); return; @@ -226,7 +226,7 @@ void PythonServiceHandler::getFileDiagram( const int32_t diagramId_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif PythonDiagram diagram(_db, _datadir, _context); @@ -252,7 +252,7 @@ void PythonServiceHandler::getFileDiagramLegend( const std::int32_t diagramId_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif PythonDiagram diagram(_db, _datadir, _context); @@ -278,7 +278,7 @@ void PythonServiceHandler::getReferenceTypes( const core::AstNodeId& astNodeId) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; #endif return_.emplace("Definition", DEFINITION); return_.emplace("Usage", USAGE); @@ -318,7 +318,7 @@ void PythonServiceHandler::getReferences( const std::vector& tags_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; LOG(info) << "astNodeID: " << astNodeId_; #endif std::vector nodes = PythonServiceHandler::queryReferences(astNodeId_, referenceId_); @@ -338,7 +338,7 @@ std::int32_t PythonServiceHandler::getReferenceCount( const std::int32_t referenceId_) { #ifndef NDEBUG - LOG(info) << "[PYTHONSERVICE]" << __func__; + LOG(info) << "[PYTHONSERVICE] " << __func__; LOG(info) << "astNodeID: " << astNodeId_; #endif From bcb17b0bc68766594ecf167bc713d52dbb0fb65a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 24 Nov 2024 14:45:15 +0100 Subject: [PATCH 150/156] [PythonParser] parse() string ref --- plugins/python/parser/src/pythonparser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 011ddfb4e..e464aaa0e 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -173,7 +173,7 @@ void PythonParser::processFile(const python::object& obj, PYNameMap& map, ParseR bool PythonParser::parse() { - for(std::string path : _ctx.options["input"].as>()) + for(const std::string& path : _ctx.options["input"].as>()) { PythonParser::parseProject(path); } From 51dcc772102878d414b72326901b5d48c9a389a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 24 Nov 2024 14:59:45 +0100 Subject: [PATCH 151/156] [PyParser] ASTHelper __equalPos --- plugins/python/parser/pyparser/asthelper.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/plugins/python/parser/pyparser/asthelper.py b/plugins/python/parser/pyparser/asthelper.py index d5f6e2630..2fe495690 100644 --- a/plugins/python/parser/pyparser/asthelper.py +++ b/plugins/python/parser/pyparser/asthelper.py @@ -55,15 +55,18 @@ def __getClasses(self) -> List[ast.ClassDef]: def __getImports(self) -> List[ast.Import | ast.ImportFrom]: return cast(List[ast.Import | ast.ImportFrom], list(filter(lambda e : isinstance(e, ast.Import) or isinstance(e, ast.ImportFrom), self.astNodes))) + def __equalPos(self, e: ast.expr | ast.stmt, pos: PosInfo): + return (e.lineno == pos.line_start and + e.end_lineno == pos.line_end and + e.col_offset == pos.column_start and + e.end_col_offset == pos.column_end) + def isFunctionCall(self, pos: PosInfo): for e in self.calls: func = e.func if (isinstance(func, ast.Name)): - if (func.lineno == pos.line_start and - func.end_lineno == pos.line_end and - func.col_offset == pos.column_start and - func.end_col_offset == pos.column_end): + if (self.__equalPos(func, pos)): return True elif (isinstance(func, ast.Attribute)): @@ -82,10 +85,7 @@ def isFunctionCall(self, pos: PosInfo): def isImport(self, pos: PosInfo): for e in self.imports: - if (e.lineno == pos.line_start and - e.end_lineno == pos.line_end and - e.col_offset == pos.column_start and - e.end_col_offset == pos.column_end): + if (self.__equalPos(e, pos)): return True return False @@ -247,10 +247,7 @@ def getFunctionSignatureByPosition(self, pos: PosInfo) -> str | None: isinstance(func.end_col_offset, int)): continue - if not (func.lineno == pos.line_start and - func.end_lineno == pos.line_end and - func.col_offset == pos.column_start and - func.end_col_offset == pos.column_end): + if not (self.__equalPos(func, pos)): continue return self.__getFunctionSignature(func) From 706b7e64ad04758e2d93300c479f3dbff66a655c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 24 Nov 2024 18:07:35 +0100 Subject: [PATCH 152/156] [Diagrams] Use relative path --- plugins/python/service/src/diagram.cpp | 29 +++++++++++++++++++++++--- plugins/python/service/src/diagram.h | 1 + 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/plugins/python/service/src/diagram.cpp b/plugins/python/service/src/diagram.cpp index 5be893a0d..36a978b36 100644 --- a/plugins/python/service/src/diagram.cpp +++ b/plugins/python/service/src/diagram.cpp @@ -196,7 +196,7 @@ util::Graph::Subgraph PythonDiagram::getFileSubgraph(util::Graph& graph_, const const std::string pathColor = (pyname.is_builtin && pyname.is_definition) ? "dodgerblue" : "limegreen"; const std::string coloredLabel = "
" + - fileInfo.path + + getRelativePath(fileInfo.path) + "
"; graph_.setSubgraphAttribute(subgraph, "label", coloredLabel, true); @@ -226,7 +226,8 @@ util::Graph::Node PythonDiagram::addPYNameNode(util::Graph& graph_, const model: util::Graph::Node PythonDiagram::addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo) { util::Graph::Node node = graph_.getOrCreateNode("f" + fileInfo.id); - graph_.setNodeAttribute(node, "label", fileInfo.path); + const std::string path = getRelativePath(fileInfo.path); + graph_.setNodeAttribute(node, "label", path); return node; } @@ -242,7 +243,8 @@ util::Graph::Node PythonDiagram::addFileNode(util::Graph& graph_, const core::Fi const std::string id = (nodeType == ImportedFilePathNode || nodeType == ImportedBuiltinFilePathNode) ? "d" + fileInfo.id : "s" + fileInfo.id; util::Graph::Node node = graph_.getOrCreateNode(id); - graph_.setNodeAttribute(node, "label", fileInfo.path); + const std::string path = getRelativePath(fileInfo.path); + graph_.setNodeAttribute(node, "label", path); decorateNode(graph_, node, nodeType); if (nodeType == ImportedFilePathNode || nodeType == ImportedBuiltinFilePathNode) { @@ -494,6 +496,27 @@ std::string PythonDiagram::getClassTable(const model::PYName& pyname) label += ""; return label; } + +std::string PythonDiagram::getRelativePath(const std::string& path) +{ + std::map labels; + m_projectService.getLabels(labels); + + if (labels.count("src")) { + std::string projectPath = labels["src"]; + std::string projectPathWithSlash = labels["src"] + "/"; + + if (path.substr(0, projectPathWithSlash.size()) == projectPathWithSlash) { + return path.substr(projectPathWithSlash.size()); + } + + if (path.substr(0, projectPath.size()) == projectPath) { + return path.substr(projectPath.size()); + } + } + + return path; +} } // language } // service } // cc diff --git a/plugins/python/service/src/diagram.h b/plugins/python/service/src/diagram.h index 8083cf326..6a1e1385b 100644 --- a/plugins/python/service/src/diagram.h +++ b/plugins/python/service/src/diagram.h @@ -57,6 +57,7 @@ namespace language util::Graph::Node addFileNode(util::Graph& graph_, const core::FileInfo& fileInfo); util::Graph::Subgraph getFileSubgraph(util::Graph& graph_, const model::PYName& pyname); std::string getClassTable(const model::PYName& pyname); + std::string getRelativePath(const std::string& path); }; } // language } // service From e5606327443c274f40066b33810b60407efa8282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Sun, 1 Dec 2024 11:14:35 +0100 Subject: [PATCH 153/156] [PythonService] Remove getProperties message for release builds --- plugins/python/service/src/pythonservice.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 81307d8df..0053441f2 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -83,7 +83,9 @@ void PythonServiceHandler::getProperties( std::map& return_, const core::AstNodeId& astNodeId_) { +#ifndef NDEBUG LOG(info) << "[PYTHONSERVICE] " << __func__; +#endif model::PYName pyname = PythonServiceHandler::queryNodeByID(astNodeId_); if(!pyname.full_name.empty()) From 286ad16fd90a8a834dde6dc2bcd92ec9f4044ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Tue, 25 Feb 2025 17:12:38 +0100 Subject: [PATCH 154/156] Fix compilation warnings. --- .../service/include/service/pythonservice.h | 42 +++++++++---------- plugins/python/service/src/pythonservice.cpp | 4 +- util/src/graph.cpp | 3 ++ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index ae6662e66..efbb7aca0 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -49,8 +49,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf const core::AstNodeId& astNodeId_) override; void getDocumentation( - std::string& return_, - const core::AstNodeId& astNodeId_) override {}; + std::string&, + const core::AstNodeId&) override {}; void getProperties( std::map& return_, @@ -97,35 +97,35 @@ class PythonServiceHandler : virtual public LanguageServiceIf const std::int32_t referenceId_) override; void getReferencesInFile( - std::vector& return_, - const core::AstNodeId& astNodeId_, - const std::int32_t referenceId_, - const core::FileId& fileId_, - const std::vector& tags_) override {}; + std::vector&, + const core::AstNodeId&, + const std::int32_t, + const core::FileId&, + const std::vector&) override {}; void getReferencesPage( - std::vector& return_, - const core::AstNodeId& astNodeId_, - const std::int32_t referenceId_, - const std::int32_t pageSize_, - const std::int32_t pageNo_) override {}; + std::vector&, + const core::AstNodeId&, + const std::int32_t, + const std::int32_t, + const std::int32_t) override {}; void getFileReferenceTypes( - std::map& return_, - const core::FileId& fileId_) override {}; + std::map&, + const core::FileId&) override {}; void getFileReferences( - std::vector& return_, - const core::FileId& fileId_, - const std::int32_t referenceId_) override {}; + std::vector&, + const core::FileId&, + const std::int32_t) override {}; std::int32_t getFileReferenceCount( - const core::FileId& fileId_, - const std::int32_t referenceId_) override {}; + const core::FileId&, + const std::int32_t) override { return 0; }; void getSyntaxHighlight( - std::vector& return_, - const core::FileRange& range_) override {}; + std::vector&, + const core::FileRange&) override {}; enum ReferenceType { diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 0053441f2..0d28e4bf8 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -213,7 +213,7 @@ void PythonServiceHandler::getDiagramLegend( void PythonServiceHandler::getFileDiagramTypes( std::map& return_, - const core::FileId& fileId_) + const core::FileId&) { #ifndef NDEBUG LOG(info) << "[PYTHONSERVICE] " << __func__; @@ -317,7 +317,7 @@ void PythonServiceHandler::getReferences( std::vector& return_, const core::AstNodeId& astNodeId_, const std::int32_t referenceId_, - const std::vector& tags_) + const std::vector&) { #ifndef NDEBUG LOG(info) << "[PYTHONSERVICE] " << __func__; diff --git a/util/src/graph.cpp b/util/src/graph.cpp index 19f315623..155106201 100644 --- a/util/src/graph.cpp +++ b/util/src/graph.cpp @@ -289,6 +289,9 @@ std::string Graph::output(Graph::Format format_) const case Graph::CAIRO_SVG: render_format = "svg:cairo"; break; + default: + __builtin_unreachable(); + break; } gvRenderData( From c946ed85a05c9d6b8f248f08449cbb6792e6aeca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 27 Feb 2025 13:36:19 +0100 Subject: [PATCH 155/156] Added file refs option. --- plugins/python/parser/pyparser/parser.py | 3 ++- plugins/python/parser/pyparser/parserconfig.py | 2 +- plugins/python/parser/pyparser/parserlog.py | 5 +++++ plugins/python/parser/src/pythonparser.cpp | 3 +++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/python/parser/pyparser/parser.py b/plugins/python/parser/pyparser/parser.py index 770126780..3201c3eb4 100644 --- a/plugins/python/parser/pyparser/parser.py +++ b/plugins/python/parser/pyparser/parser.py @@ -26,7 +26,8 @@ def parseProject(settings, n_proc): ast_import=settings["ast_import"], ast_annotations=settings["ast_annotations"], ast_inheritance=settings["ast_inheritance"], - ast_function_signature=settings["ast_function_signature"] + ast_function_signature=settings["ast_function_signature"], + file_refs=settings["file_refs"] ) log(f"Parsing project: {config.root_path}") diff --git a/plugins/python/parser/pyparser/parserconfig.py b/plugins/python/parser/pyparser/parserconfig.py index e6927c845..1f59b7833 100644 --- a/plugins/python/parser/pyparser/parserconfig.py +++ b/plugins/python/parser/pyparser/parserconfig.py @@ -12,7 +12,7 @@ class ParserConfig: stack_trace: bool = False safe_env: bool = False type_hint: bool = False - file_refs: bool = True + file_refs: bool = False submodule_discovery: bool = True ast: bool = True ast_function_call: bool = True diff --git a/plugins/python/parser/pyparser/parserlog.py b/plugins/python/parser/pyparser/parserlog.py index 2bec87be9..1c0dcc73a 100644 --- a/plugins/python/parser/pyparser/parserlog.py +++ b/plugins/python/parser/pyparser/parserlog.py @@ -25,6 +25,11 @@ def log_config(config: ParserConfig): else: log(f"{bcolors.OKBLUE}Type hint support disabled!") + if config.file_refs: + log(f"{bcolors.OKGREEN}File refs lookup enabled!") + else: + log(f"{bcolors.OKBLUE}File refs lookup disabled!") + if config.stack_trace: log(f"{bcolors.WARNING}Stack trace enabled!") diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index e464aaa0e..f52c8703b 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -67,6 +67,7 @@ void PythonParser::parseProject(const std::string& root_path) settings["ast_annotations"] = !(_ctx.options.count("disable-ast-annotations")); settings["ast_inheritance"] = !(_ctx.options.count("disable-ast-inheritance")); settings["ast_function_signature"] = !(_ctx.options.count("disable-ast-function-signature")); + settings["file_refs"] = (bool)(_ctx.options.count("file-refs")); python::object result_list = m_py_module.attr("parseProject")(settings, n_proc); @@ -215,6 +216,8 @@ extern "C" "Disable AST inheritance parsing.") ("disable-ast-function-signature", "Disable AST function signature parsing.") + ("file-refs", + "Enable search for references in a file context.") ("debug", "Enable parsing in debug mode."); From a55f31c96a528b8003e223a8d99bcd79422ea58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20Domozi?= Date: Thu, 27 Feb 2025 13:36:53 +0100 Subject: [PATCH 156/156] Added Python plugin documentation. --- doc/pythonplugin.md | 123 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 doc/pythonplugin.md diff --git a/doc/pythonplugin.md b/doc/pythonplugin.md new file mode 100644 index 000000000..f1e4442d0 --- /dev/null +++ b/doc/pythonplugin.md @@ -0,0 +1,123 @@ +# Python Plugin + +## Parsing Python projects +Python projects can be parsed by using the `CodeCompass_parser` executable. +See its usage [in a seperate document](/doc/usage.md). + +## Python specific parser flags + +### Python dependencies +Large Python projects usually have multiple Python package dependencies. +Although a given project can be parsed without installing any of its dependencies, it is strongly recommended +that the required modules are installed in order to achieve a complete parsing. +To install a project's dependencies, create a [Python virtual environment](https://docs.python.org/3/library/venv.html) +and install the necessary packages. +When parsing a project, specify the virtual environment path so the parser can successfully resolve the dependencies: +``` +--venvpath +``` + +### Type hints +The parser can try to determine Python type hints for variables, expressions and functions. +It can work out type hints such as `Iterable[int]` or `Union[int, str]`. +However, this process can be extremely slow, especially for functions, thus it is disabled by default. +It can be enabled using the `--type-hint` flag. + +### Python submodules +Large Python projects can have internal submodules and the parser tries to locate them automatically. +Specifically, it looks for `__init__.py` files and considers those folders modules. +This process is called submodule discovery and can be disabled using the `--disable-submodule-discovery` flag. + +You can also add submodules manually by adding those specific paths to the parser's syspath: +``` +--syspath +``` +For more information, see the [Python syspath docs](https://docs.python.org/3/library/sys.html#sys.path). + +### File references +By default, the parser works out references by looking for definitions only - if nodes share the same definition +they are considered references. +However, this method sometimes misses a few references (e.g. local variables in a function). +To extend search for references in a file context, apply the `--file-refs` flag. +Note that using this option can potentially extend the total parsing time. + +## Examples of parsing Python projects + +### Flask +We downloaded [flask 3.1.0](https://github.com/pallets/flask/releases/tag/3.1.0) source code to `~/parsing/flask/`. +The first step is to create a Python virtual environment and install flask's dependencies. +Create a Python virtual environment and activate it: +```bash +cd ~/parsing/flask/ +python3 -m venv venv +source venv/bin/activate +``` +Next, we install the required dependencies listed in `pyproject.toml`. +```bash +pip install . +``` +Further dependencies include development packages listed in `requirements/dev.txt`. +These can be also installed using `pip`. +```bash +pip install -r requirements/dev.txt +``` +Finally, we can run `CodeCompass_parser`. +```bash +CodeCompass_parser \ + -n flask \ + -i ~/parsing/flask/ \ + -w ~/parsing/workdir/ \ + -d "pgsql:host=localhost;port=5432;user=compass;password=pass;database=flask" \ + -f \ + --venvpath ~/parsing/flask/venv/ \ + --label src=~/parsing/flask/ +``` + +### CodeChecker +We downloaded [CodeChecker 6.24.4](https://github.com/Ericsson/codechecker/releases/tag/v6.24.4) source code to `~/parsing/codechecker`. +CodeChecker has an automated way of creating a Python virtual environment and installing dependencies - by running the `venv` target of a Makefile: +```bash +cd ~/parsing/codechecker/ +make venv +``` +Next, we can run `CodeCompass_parser`. +```bash +CodeCompass_parser \ + -n codechecker \ + -i ~/parsing/codechecker/ \ + -w ~/parsing/workdir/ \ + -d "pgsql:host=localhost;port=5432;user=compass;password=pass;database=codechecker" \ + -f \ + --venvpath ~/parsing/codechecker/venv/ \ + --label src=~/parsing/codechecker/ +``` + +## Troubleshooting +A few errors can occur during the parsing process, these are highlighted in color red. +The stack trace is hidden by default, and can be shown using the `--stack-trace` flag. + +### Failed to use virtual environment +This error can appear if one specifies the `--venvpath` option during parsing. +The parser tried to use the specified virtual environment path, however it failed. + +#### Solution +Double check that the Python virtual environment is correctly setup and its +path is correct. +If the error still persists, apply the `--stack-trace` parser option +to view a more detailed stack trace of the error. + +### Missing module (file = path line = number) +In this case, the parser tried to parse a given Python file, however it +could not find a definition for a module. +Commonly, the Python file imports another module and the parser cannot locate this module. +If this happens, the Python file is marked *partial* indicating that +a module definition was not resolved in this file. +The error message displays the module name, exact file path and line number +so one can further troubleshoot this problem. + +#### Solution +Ensure that the `--venvpath` option is correctly specified and all the required +dependencies are installed in that Python virtual environment. +If the imported module is part of the parsed project, use the `--syspath` option +and specify the directory where the module is located in. +