Merge pull request #260 from JohT/feature/improve-external-modules-and-declarations-naming

JohT · web-flow · commit ff3a39035ab6 · 2024-11-04T20:10:25.000+01:00
Improve naming of external Typescript nodes
diff --git a/.github/workflows/java-code-analysis.yml b/.github/workflows/java-code-analysis.yml
@@ -48,7 +48,7 @@ jobs:
         - os: ubuntu-latest
           java: 17
           python: 3.11
-          mambaforge: 24.9.0-0
+          miniforge: 24.9.0-0
 
     env: 
       CI_COMMIT_MESSAGE: Automated code structure analysis reports (CI)
@@ -78,7 +78,7 @@ jobs:
       working-directory: graph-visualization
       run: npm ci
 
-    - name: Setup Cache for Conda package manager Mambaforge
+    - name: Setup Cache for Conda package manager Miniforge
       uses: actions/cache@v4
       env:
         # Increase this value to reset cache if etc/example-environment.yml has not changed
@@ -90,13 +90,11 @@ jobs:
           ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-environments-${{hashFiles('**/environment.yml', '.github/workflows/*.yml') }}
 
     # "Setup Python" can be skipped if jupyter notebook reports aren't needed
-    - name: Setup Python ${{ matrix.python }} with Conda package manager Mambaforge
+    - name: Setup Python ${{ matrix.python }} with Conda package manager Miniforge
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: ${{ matrix.python }}
-        miniforge-variant: Mambaforge
-        miniforge-version: ${{ matrix.mambaforge }}
-        use-mamba: true
+        miniforge-version: ${{ matrix.miniforge }}
         activate-environment: codegraph
         environment-file: ./jupyter/environment.yml
         auto-activate-base: false
@@ -144,7 +142,7 @@ jobs:
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
+        name: code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
         path: |
           ./temp/**/runtime/*
           ./temp/**/reports/*
@@ -155,7 +153,7 @@ jobs:
       if: success()
       uses: actions/upload-artifact@v4
       with:
-        name: code-report-results-java-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
+        name: code-report-results-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
         path: ./results
         if-no-files-found: error
         retention-days: 5
@@ -166,7 +164,7 @@ jobs:
     #- name: Archive exported database
     #  uses: actions/upload-artifact@v3
     #  with:
-    #    name: code-report-database-export-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
+    #    name: code-report-database-export-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
     #    path: ./temp/**/import
     #    if-no-files-found: error
     #    retention-days: 5
diff --git a/.github/workflows/typescript-code-analysis.yml b/.github/workflows/typescript-code-analysis.yml
@@ -48,7 +48,7 @@ jobs:
         - os: ubuntu-latest
           java: 17
           python: 3.11
-          mambaforge: 24.9.0-0
+          miniforge: 24.9.0-0
 
     env: 
       CI_COMMIT_MESSAGE: Automated code structure analysis reports (CI)
@@ -69,7 +69,7 @@ jobs:
         distribution: 'adopt'
         java-version: ${{ matrix.java }}
 
-    - name: Setup Node.js
+    - name: Setup Node.js for Graph Visualization
       uses: actions/setup-node@v4
       with:
         node-version-file: 'graph-visualization/.nvmrc'
@@ -78,7 +78,7 @@ jobs:
       working-directory: graph-visualization
       run: npm ci
 
-    - name: Setup Cache for Conda package manager Mambaforge
+    - name: Setup Cache for Conda package manager Miniforge
       uses: actions/cache@v4
       env:
         # Increase this value to reset cache if etc/example-environment.yml has not changed
@@ -90,13 +90,11 @@ jobs:
           ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-environments-${{hashFiles('**/environment.yml', '.github/workflows/*.yml') }}
 
     # "Setup Python" can be skipped if jupyter notebook reports aren't needed
-    - name: Setup Python ${{ matrix.python }} with Conda package manager Mambaforge
+    - name: Setup Python ${{ matrix.python }} with Conda package manager Miniforge
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: ${{ matrix.python }}
-        miniforge-variant: Mambaforge
-        miniforge-version: ${{ matrix.mambaforge }}
-        use-mamba: true
+        miniforge-version: ${{ matrix.miniforge }}
         activate-environment: codegraph
         environment-file: ./jupyter/environment.yml
         auto-activate-base: false
@@ -153,7 +151,7 @@ jobs:
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
+        name: code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
         path: |
           ./temp/**/runtime/*
           ./temp/**/reports/*
@@ -164,7 +162,7 @@ jobs:
       if: success()
       uses: actions/upload-artifact@v4
       with:
-        name: code-report-results-java-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
+        name: code-report-results-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
         path: ./results
         if-no-files-found: error
         retention-days: 5
@@ -175,7 +173,7 @@ jobs:
     #- name: Archive exported database
     #  uses: actions/upload-artifact@v3
     #  with:
-    #    name: code-report-database-export-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
+    #    name: code-report-database-export-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
     #    path: ./temp/**/import
     #    if-no-files-found: error
     #    retention-days: 5
diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md
@@ -8,7 +8,35 @@ For more details on how the commands work in detail see [COMMANDS](./COMMANDS.md
 
 Please read through the [Prerequisites](./README.md#hammer_and_wrench-prerequisites) in the [README](./README.md) file for what is required to run the scripts.
 
-## Start an analysis
+## The easiest way to get started
+
+Just run one of the following examples in the directory of this file:
+
+- [./scripts/examples/analyzeAxonFramework.sh](./scripts/examples/analyzeAxonFramework.sh) (Java event-sourcing library)
+- [./scripts/examples/analyzeAntDesign.sh](./scripts/examples/analyzeAntDesign.sh) (Typescript UI library)
+- [./scripts/examples/analyzeReactRouter.sh](./scripts/examples/analyzeReactRouter.sh) (Typescript React library)
+
+Use these optional command line options as you like:
+
+- (Recommended) Only create CSV reports and skip Python and Node.js dependent reports. Example:
+
+  ```shell
+  ./scripts/examples/analyzeAxonFramework.sh --report CSV
+  ```
+
+- Only explore the graph manually in the [browser](http://localhost:7474/browser). Skip all automated reports. Example:
+
+  ```shell
+  ./scripts/examples/analyzeAxonFramework.sh --explore
+  ```
+
+- Add the version number of the project to pick a specific one. Example:
+
+  ```shell
+  ./scripts/examples/analyzeAxonFramework.sh 4.10.1
+  ```
+
+## Start an own analysis
 
 1. Create a directory for all analysis projects.
 
diff --git a/cypher/Dependencies_Projection/Dependencies_0_Verify_Projectable.cypher b/cypher/Dependencies_Projection/Dependencies_0_Verify_Projectable.cypher
@@ -7,7 +7,7 @@
       ,(dependency[$dependencies_projection_weight_property])             AS weightPropertyValue
       ,(dependency[$dependencies_projection_weight_property] < 1)         AS nonPositiveWeightPropertyValue
       ,coalesce(dependency.resolved, false)                               AS resolvedDependency
-      ,EXISTS { (target)<-[:RESOLVES_TO]-(resolvedTarget:ExternalModule) } AS resolvedTarget
+      ,EXISTS { (target)<-[:IS_IMPLEMENTED_IN]-(resolvedTarget:ExternalModule) } AS resolvedTarget
       ,(source.incomingDependencies IS NULL OR
         target.incomingDependencies IS NULL)   AS missingIncomingDependencies
       ,(source.outgoingDependencies IS NULL OR 
diff --git a/cypher/DependsOn_Relationship_Weights/Add_fine_grained_weights_for_Typescript_external_module_dependencies.cypher b/cypher/DependsOn_Relationship_Weights/Add_fine_grained_weights_for_Typescript_external_module_dependencies.cypher
@@ -4,8 +4,8 @@
  MATCH (source:TS:Module)-[moduleDependency:DEPENDS_ON]->(target:ExternalModule)
 // Exclude all targets where an ExternalModule was found that resolves to them
 // because those are covered in the fine grained weights for "ExternalModule"s.
- WHERE NOT EXISTS { (target)-[:RESOLVES_TO]->(source) }
-OPTIONAL MATCH (source)-[resolvedModuleDependency:DEPENDS_ON]->(resolvedTarget:TS:Module)<-[:RESOLVES_TO]-(target)
+ WHERE NOT EXISTS { (target)-[:IS_IMPLEMENTED_IN]->(source) }
+OPTIONAL MATCH (source)-[resolvedModuleDependency:DEPENDS_ON]->(resolvedTarget:TS:Module)<-[:IS_IMPLEMENTED_IN]-(target)
   WITH source
       ,target
       ,moduleDependency
@@ -24,7 +24,7 @@ OPTIONAL MATCH (source)-[rd:DEPENDS_ON]->(declaration:ExternalDeclaration)<-[:EX
       ,collect(declaration)                          AS declarations
 // Get optional low coupling elements (TypeAlias, Interface) that the source module contains and defines (low level) that depend on the external module (target)
 UNWIND declarations AS declaration
-OPTIONAL MATCH (source)-[ra:DEPENDS_ON]->(declaration)-[:RESOLVES_TO]->(abstractType:TypeAlias|Interface)
+OPTIONAL MATCH (source)-[ra:DEPENDS_ON]->(declaration)-[:IS_IMPLEMENTED_IN]->(abstractType:TypeAlias|Interface)
   WITH source
       ,target
       ,moduleDependency
diff --git a/cypher/DependsOn_Relationship_Weights/Add_fine_grained_weights_for_Typescript_internal_module_dependencies.cypher b/cypher/DependsOn_Relationship_Weights/Add_fine_grained_weights_for_Typescript_internal_module_dependencies.cypher
@@ -7,7 +7,7 @@
  WHERE moduleDependency.declarationCount IS NULL
 // Ruling out resolved targets also filters out entries that aren't covered by the fine grained weights for "ExternalModule"s.
 // Therefore, the exists filter is commented out for now and replaced by focussing on missing detailed weight properties to catch them all.
-//WHERE NOT EXISTS { (target)<-[:RESOLVES_TO]-(resolvedTarget:ExternalModule) }
+//WHERE NOT EXISTS { (target)<-[:IS_IMPLEMENTED_IN]-(resolvedTarget:ExternalModule) }
   WITH source
       ,target
       ,moduleDependency
diff --git a/cypher/External_Dependencies/List_external_modules_resolved_to_internal_ones_for_Typescript.cypher b/cypher/External_Dependencies/List_external_modules_resolved_to_internal_ones_for_Typescript.cypher
@@ -1,6 +1,6 @@
 // Statistics about how many ExternalModule nodes were found that match internal Module nodes 
 
- MATCH (module:TS:Module)<-[resolved:RESOLVES_TO]-(external:TS:ExternalModule)
+ MATCH (module:TS:Module)<-[resolved:IS_IMPLEMENTED_IN]-(external:TS:ExternalModule)
 OPTIONAL MATCH (project:TS:Project)-[:CONTAINS]->(module)
   WITH project.name                               AS projectName
       ,count(DISTINCT module)                     AS resolvedModuleCount
diff --git a/cypher/GitLog/Delete_plain_git_directory_file_nodes.cypher b/cypher/GitLog/Delete_plain_git_directory_file_nodes.cypher
@@ -1,6 +1,6 @@
 // Delete plain file nodes in "/.git" directory
 
-MATCH (git_metadata_file:File)<-[:CONTAINS*]-(git_directory:Directory)
+MATCH (git_metadata_file:File&!Repository)<-[:CONTAINS*]-(git_directory:Directory&!Repository)
 WHERE git_directory.fileName ENDS WITH '/.git'
  WITH git_directory.fileName AS gitDirectory
      ,count(DISTINCT git_metadata_file.fileName)          AS numberOfFiles
diff --git a/cypher/Metrics/Set_Outgoing_Typescript_Module_Dependencies.cypher b/cypher/Metrics/Set_Outgoing_Typescript_Module_Dependencies.cypher
@@ -3,9 +3,9 @@
 // Get the top level dependency between a Typescript module and an external modules it uses
  MATCH (source:TS:Module)
 OPTIONAL MATCH (source)-[moduleDependency:DEPENDS_ON]->(target:ExternalModule)
- WHERE NOT EXISTS {(target)-[:RESOLVES_TO]->(source)}
+ WHERE NOT EXISTS {(target)-[:IS_IMPLEMENTED_IN]->(source)}
 // Get the project of the external module if available
-OPTIONAL MATCH (projectdir:Directory)<-[:HAS_ROOT]-(project:TS:Project)-[:CONTAINS]->(:TS:Module)<-[:RESOLVES_TO]-(target)
+OPTIONAL MATCH (projectdir:Directory)<-[:HAS_ROOT]-(project:TS:Project)-[:CONTAINS]->(:TS:Module)<-[:IS_IMPLEMENTED_IN]-(target)
 // Aggregate all gathered information for each (grouped by) source module
   WITH source
       ,collect(DISTINCT projectdir.absoluteFileName) AS projectNames
diff --git a/cypher/Typescript_Enrichment/Add_DEPENDS_ON_relationship_to_resolved_modules.cypher b/cypher/Typescript_Enrichment/Add_DEPENDS_ON_relationship_to_resolved_modules.cypher
@@ -2,7 +2,7 @@
 // Inspired by https://github.com/jQAssistant/jqassistant/blob/4cd7face5d6d2953449d8e6ff5b484f00ffbdc2f/plugin/java/src/main/resources/META-INF/jqassistant-rules/java-classpath.xml#L5
 
  MATCH (module:TS:Module)-[dependsOn:DEPENDS_ON]->(externalModule:TS:ExternalModule)
- MATCH (externalModule)-[:RESOLVES_TO]->(resolvedModule:TS:Module)
+ MATCH (externalModule)-[:IS_IMPLEMENTED_IN]->(resolvedModule:TS:Module)
  WHERE module <> resolvedModule
   CALL {  WITH module, dependsOn, resolvedModule
          MERGE (module)-[resolvedDependsOn:DEPENDS_ON]->(resolvedModule)
diff --git a/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_declarations.cypher b/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_declarations.cypher
@@ -0,0 +1,18 @@
+// Link matching external to internal Typescript declarations with an IS_IMPLEMENTED_IN relationship
+
+ MATCH (externalModule:TS&ExternalModule)-[:EXPORTS]->(externalDeclaration:TS&ExternalDeclaration)
+ MATCH (externalModule)-[:IS_IMPLEMENTED_IN]->(internalModule:TS&Module)
+ MATCH (externalModule)-[:EXPORTS]->(internalDeclaration:TS&!ExternalDeclaration)
+ WHERE externalDeclaration.name = internalDeclaration.name
+  WITH externalDeclaration, internalDeclaration
+  CALL { WITH externalDeclaration, internalDeclaration
+        MERGE (externalDeclaration)-[:IS_IMPLEMENTED_IN]->(internalDeclaration)
+       } IN TRANSACTIONS
+RETURN count(  DISTINCT externalDeclaration.globalFqn + ' -> ' + internalDeclaration.globalFqn)       AS linkedDeclarationCount
+      ,collect(DISTINCT externalDeclaration.globalFqn + ' -> ' + internalDeclaration.globalFqn)[0..4] AS linkedDeclarationExamples
+//Debugging
+//RETURN split(internalDeclaration.globalFqn, '/')[-6..] AS shortInternalDeclaration
+//      ,split(externalDeclaration.globalFqn, '/')[-6..] AS shortExternalDeclaration
+//      ,count(*)
+//ORDER BY shortInternalDeclaration
+//LIMIT 30
diff --git a/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_modules.cypher b/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_modules.cypher
@@ -1,4 +1,4 @@
-// Adds a relation "RESOLVES_TO" from an external module to a module if their global fully qualified names match.
+// Adds a relation "IS_IMPLEMENTED_IN" from an external module to a module if their global fully qualified names match.
 // Depends on "Add_module_properties.cypher" to be run first
 // Inspired by https://github.com/jQAssistant/jqassistant/blob/4cd7face5d6d2953449d8e6ff5b484f00ffbdc2f/plugin/java/src/main/resources/META-INF/jqassistant-rules/java-classpath.xml#L5
 // Related to https://github.com/jqassistant-plugin/jqassistant-typescript-plugin/issues/35
@@ -47,7 +47,7 @@ WHERE equalGlobalFqn
    OR equalNameWithoutNamespace
    OR equalNameAndNpmPackage
  CALL {  WITH module, externalModule
-        MERGE (externalModule)-[:RESOLVES_TO]->(module)
+        MERGE (externalModule)-[:IS_IMPLEMENTED_IN]->(module)
       } IN TRANSACTIONS
 RETURN CASE WHEN equalGlobalFqn            THEN 'equalGlobalFqn'
             WHEN equalModule               THEN 'equalModule'
diff --git a/cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationship_for_matching_declarations.cypher b/cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationship_for_matching_declarations.cypher
diff --git a/cypher/Typescript_Enrichment/Add_module_properties.cypher b/cypher/Typescript_Enrichment/Add_module_properties.cypher
@@ -5,6 +5,18 @@ OPTIONAL MATCH (class:TS:Class)-[:DECLARES]->(ts)
   WITH *
       //In case of a path like ".../moduleName/src/index.ts", the module name is extracted into sourceIndexModuleName.
       ,reverse(split(reverse(nullif(split(ts.globalFqn, '/src/index.')[0], ts.globalFqn)), '/')[0]) AS sourceIndexModuleName
+  WITH *
+      //In case of a path like ".../moduleName/dist/types/index.d.ts", the module name is extracted into distTypesIndexModuleName.
+      ,reverse(split(reverse(nullif(split(ts.globalFqn, '/dist/types/index.')[0], ts.globalFqn)), '/')[0]) AS distTypesIndexModuleName
+  WITH *
+      //In case of a path like ".../moduleName/types/index.d.ts", the module name is extracted into typesIndexModuleName.
+      ,reverse(split(reverse(nullif(split(ts.globalFqn, '/types/index.')[0], ts.globalFqn)), '/')[0]) AS typesIndexModuleName
+  WITH *
+      //In case of a path like ".../moduleName/dist/types/src/index.d.ts", the module name is extracted into distTypesSourceIndexModuleName.
+      ,reverse(split(reverse(nullif(split(ts.globalFqn, '/dist/types/src/index.')[0], ts.globalFqn)), '/')[0]) AS distTypesSourceIndexModuleName
+  WITH *
+      //Combine the cases above into "normalizedModuleName"
+      ,coalesce(distTypesIndexModuleName, typesIndexModuleName, distTypesSourceIndexModuleName, sourceIndexModuleName) AS normalizedModuleName
   WITH *
       ,replace(split(ts.globalFqn, '".')[0],'"', '')                                        AS modulePathName
       ,reverse(split(reverse(replace(split(ts.globalFqn, '".')[0],'"', '')), '/')[0])       AS moduleName
@@ -24,7 +36,7 @@ OPTIONAL MATCH (class:TS:Class)-[:DECLARES]->(ts)
     SET ts.namespace          = coalesce(nullif(namespaceNameWithAtPrefixed, ''), ts.namespace, '')
        ,ts.module             = modulePathNameWithoutIndexAndDefault
        ,ts.moduleName         = moduleName
-       ,ts.name               = coalesce(sourceIndexModuleName, symbolNameWithoutClassName, indexAndExtensionOmittedName)
+       ,ts.name               = coalesce(symbolNameWithoutClassName, normalizedModuleName, indexAndExtensionOmittedName)
        ,ts.extensionExtended  = moduleNameExtensionExtended
        ,ts.extension          = moduleNameExtension
        ,ts.isNodeModule       = isNodeModule
@@ -33,12 +45,18 @@ OPTIONAL MATCH (class:TS:Class)-[:DECLARES]->(ts)
        ,ts.packageName        = packageName
 RETURN count(*) AS updatedModules
 // For debugging
-// RETURN namespaceNameWithAtPrefixed         AS namespace
-//        ,modulePathName                     AS module            
-//        ,moduleName                         AS moduleName        
-//        ,coalesce(sourceIndexModuleName, symbolNameWithoutClassName, indexAndExtensionOmittedName) AS name              
-//        ,moduleNameExtensionExtended        AS extensionExtended 
-//        ,moduleNameExtension                AS extension         
+// RETURN  ts.globalFqn
+//        ,namespaceNameWithAtPrefixed         AS namespace
+//        ,modulePathName                      AS module            
+//        ,moduleName                          AS moduleName        
+//        ,coalesce(symbolNameWithoutClassName, normalizedModuleName, indexAndExtensionOmittedName) AS name              
+//        ,moduleNameExtensionExtended         AS extensionExtended 
+//        ,moduleNameExtension                 AS extension         
+//        ,modulePathNameWithoutIndexAndDefault 
+//        ,symbolName 
+//        ,optionalClassName 
+//        ,normalizedModuleName 
+//        ,symbolNameWithoutClassName 
 //        ,isNodeModule                       AS isNodeModule      
 //        ,isUnresolvedImport                 AS isUnresolvedImport
 //        ,isNodeModule OR isUnresolvedImport AS isExternalImport  
diff --git a/cypher/Typescript_Enrichment/Count_internal_modules_resolving_external_ones.cypher b/cypher/Typescript_Enrichment/Count_internal_modules_resolving_external_ones.cypher
@@ -4,7 +4,7 @@
   WITH count(internal_module)   AS totalNumberOfInternalModules
       ,collect(internal_module) AS internal_modules
  UNWIND internal_modules AS internal_module
- MATCH (external_module:TS:ExternalModule)-[:RESOLVES_TO]->(internal_module)
+ MATCH (external_module:TS:ExternalModule)-[:IS_IMPLEMENTED_IN]->(internal_module)
  WHERE NOT external_module.isNodeModule = true
 RETURN totalNumberOfInternalModules
       ,COUNT { (all_external_modules:TS:ExternalModule) 
diff --git a/scripts/examples/analyzeAntDesign.sh b/scripts/examples/analyzeAntDesign.sh
diff --git a/scripts/examples/analyzeAxonFramework.sh b/scripts/examples/analyzeAxonFramework.sh
diff --git a/scripts/examples/analyzeReactRouter.sh b/scripts/examples/analyzeReactRouter.sh
diff --git a/scripts/prepareAnalysis.sh b/scripts/prepareAnalysis.sh
