Merge pull request #203 from JohT/feature/useability-improvements

Improve useability

Author: JohT

README.md

@@ -182,6 +182,7 @@ The [Code Structure Analysis Pipeline](./.github/workflows/java-code-analysis.ym
 
 - How can i trigger a full re-scan of all artifacts?  
   👉 Delete the file `artifactsChangeDetectionHash.txt` in the `artifacts` directory.
+  👉 Delete the file `typescriptFileChangeDetectionHashFile.txt` in the `source` directory to additionally re-scan Typescript projects.
 
 - How can i enable PDF generation for Jupyter Notebooks (depends on chromium, takes more time)?  
   👉 Set environment variable `ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION` to anything except an empty string. Example:  

cypher/Dependencies_Projection/Dependencies_0_Verify_Projectable.cypher

@@ -0,0 +1,33 @@
+// Verify that nodes and relationships are complete and ready for projection
+
+ MATCH (source:TS:Module)-[dependency:DEPENDS_ON]->(target:Module)
+ WHERE $dependencies_projection_node            IN labels(source)
+   AND $dependencies_projection_node            IN labels(target)
+  WITH (NOT $dependencies_projection_weight_property IN keys(dependency)) AS missingWeightProperty
+      ,(dependency[$dependencies_projection_weight_property])             AS weightPropertyValue
+      ,(dependency[$dependencies_projection_weight_property] < 1)         AS nonPositiveWeightPropertyValue
+      ,coalesce(dependency.resolved, false)                               AS resolvedDependency
+      ,EXISTS { (target)<-[:RESOLVES_TO]-(resolvedTarget:ExternalModule) } AS resolvedTarget
+      ,(source.incomingDependencies IS NULL OR
+        target.incomingDependencies IS NULL)   AS missingIncomingDependencies
+      ,(source.outgoingDependencies IS NULL OR 
+        target.outgoingDependencies IS NULL)   AS missingOutgoingDependencies
+      ,source
+      ,target
+ WHERE missingWeightProperty
+//    OR nonPositiveWeightPropertyValue // if strict positive weights are needed 
+    OR missingIncomingDependencies
+    OR missingOutgoingDependencies
+RETURN missingWeightProperty
+      ,nonPositiveWeightPropertyValue
+      ,resolvedDependency
+      ,resolvedTarget
+      ,missingIncomingDependencies
+      ,missingOutgoingDependencies
+      ,count(*) AS numberOfRelationships
+      ,min(weightPropertyValue) AS minWeightPropertyValue
+      ,max(weightPropertyValue) AS maxWeightPropertyValue
+      ,collect(DISTINCT source.globalFqn + ' -> ' + target.globalFqn)[0..4] AS examples
+      // Output source and target nodes for troubleshooting
+      //,collect(source)[0..4]
+      //,collect(target)[0..4]

cypher/DependsOn_Relationship_Weights/Add_fine_grained_weights_for_Typescript_internal_module_dependencies.cypher

@@ -2,14 +2,16 @@
 
 // Get the top level dependency between a Typescript module and the external modules it uses
  MATCH (source:TS:Module)-[moduleDependency:DEPENDS_ON]->(target:Module)
-// Exclude all targets where an ExternalModule was found that resolves to them
-// because those are covered in the fine grained weights for "ExternalModule"s.
- WHERE NOT EXISTS { (target)<-[:RESOLVES_TO]-(resolvedTarget:ExternalModule) }
+// Exclude all that already have extended weight properties
+// for example because those were covered in the fine grained weights for "ExternalModule"s.
+ WHERE moduleDependency.declarationCount IS NULL
+// Ruling out resolved targets also filters out entries that aren't covered by the fine grained weights for "ExternalModule"s.
+// Therefore, the exists filter is commented out for now and replaced by focussing on missing detailed weight properties to catch them all.
+//WHERE NOT EXISTS { (target)<-[:RESOLVES_TO]-(resolvedTarget:ExternalModule) }
   WITH source
       ,target
       ,moduleDependency
       ,moduleDependency.cardinality AS targetModuleCardinality
-
 // Get optional external (e.g. type) declarations that the external module (target) provides and the source module uses
 OPTIONAL MATCH (source)-[elementDependency:DEPENDS_ON|EXPORTS]->(elementType:TS)<-[:EXPORTS]-(target)
   WITH source
@@ -40,13 +42,18 @@ OPTIONAL MATCH (source)-[abstractDependency:DEPENDS_ON|EXPORTS]->(abstractType:T
 // - "lowCouplingElement25PercentWeight" subtracts 75% of the weights for abstract types like Interfaces and Type aliases
 //   to compensate for their low coupling influence. Not included "high coupling" elements like Functions and Classes 
 //   remain in the weight as they were. The same applies for "lowCouplingElement10PercentWeight" but with in a stronger manner.
+//   If there are no declarations and therefore the elementTypeCardinality is zero then the original targetModuleCardinality is used.
  SET moduleDependency.declarationCount        = elementTypeCount
     ,moduleDependency.abstractTypeCount       = abstractTypeCount
     ,moduleDependency.abstractTypeCardinality = abstractTypeCardinality
-    ,moduleDependency.lowCouplingElement25PercentWeight = 
-        toInteger(elementTypeCardinality - round(abstractTypeCardinality * 0.75))
-    ,moduleDependency.lowCouplingElement10PercentWeight = 
-        toInteger(elementTypeCardinality - round(abstractTypeCardinality * 0.90))
+    ,moduleDependency.lowCouplingElement25PercentWeight = toInteger(
+        coalesce(nullif(elementTypeCardinality, 0), targetModuleCardinality) - 
+        round(abstractTypeCardinality * 0.75)
+     )
+    ,moduleDependency.lowCouplingElement10PercentWeight = toInteger(
+        coalesce(nullif(elementTypeCardinality, 0), targetModuleCardinality) - 
+        round(abstractTypeCardinality * 0.90)
+     )
 RETURN source.globalFqn    AS sourceName
       ,target.globalFqn    AS targetName
       ,elementTypeCount

cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher

@@ -1,17 +1,28 @@
-// Connect git files to code files with a RESOLVES_TO relationship if their names match
-// Note: Even if is tempting to combine this file with the Typescript variant, they are intentionally spearated.
+// Connect git files to Java code files with a RESOLVES_TO relationship if their names match
+// Note: Its quite tricky to match Java class file paths from inside e.g. *.jar files to their source repository file path reliable.
+//       This could be improved by utilizing package manager data (like maven). Even that turns out to be not easy,
+//       since the folder structure can be customized. Therefore, this is only a simplified attempt and by no means complete.
+// Note: Even if is tempting to combine this file with the Typescript variant, they are intentionally separated.
 //       The differences are subtle but need to be thought through and tested carefully.
 //       Having separate files makes it obvious that there needs to be one for every new source code language.
 
-MATCH (code_file:File&!Git)
-WHERE NOT EXISTS { (code_file)-[:RESOLVES_TO]->(other_file:File&!Git) } // only original nodes, no duplicates
- WITH code_file, replace(code_file.fileName, '.class', '.java') AS codeFileName
-MATCH (git_file:File&Git)
+MATCH (code_file:!Git&File)
+WHERE  code_file.fileName IS NOT NULL
+  // Use only original code files, no resolved duplicates
+  AND NOT EXISTS { (code_file)-[:RESOLVES_TO]->(other_file:File&!Git) }
+ WITH code_file
+     ,replace(code_file.fileName, '.class', '.java') AS codeFileName
+MATCH (git_file:Git&File)
  WITH *
      ,git_file
      ,coalesce(git_file.fileName, git_file.relativePath) AS gitFileName
 WHERE gitFileName ENDS WITH codeFileName
 MERGE (git_file)-[:RESOLVES_TO]->(code_file)
   SET git_file.resolved = true
 RETURN count(DISTINCT codeFileName)  AS numberOfCodeFiles
-      ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
+      ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
+// RETURN codeFileName, gitFileName
+//       ,git_file.fileName, git_file.relativePath
+//       ,git_repository.fileName , code_file.absoluteFileName
+//       ,git_repository.name, code_file.fileName
+// LIMIT 20

cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher

@@ -1,17 +1,30 @@
 // Connect git files to Typescript files with a RESOLVES_TO relationship if their names match
-// Note: Even if is tempting to combine this file with the Java variant, they are intentionally spearated.
+// Note: Even if is tempting to combine this file with the Java variant, they are intentionally separated.
 //       The differences are subtle but need to be thought through and tested carefully.
 //       Having separate files makes it obvious that there needs to be one for every new source code language.
 
-MATCH (code_file:File&!Git)
-WHERE NOT EXISTS { (code_file)-[:RESOLVES_TO]->(other_file:File&!Git) } // only original nodes, no duplicates
- WITH code_file, code_file.absoluteFileName AS codeFileName
-MATCH (git_file:File&Git)
+MATCH (code_file:!Git&File)
+WHERE (code_file.absoluteFileName IS NOT NULL OR code_file.fileName IS NOT NULL)
+  // Use only original code files, no resolved duplicates
+  AND NOT EXISTS { (code_file)-[:RESOLVES_TO]->(other_file:File&!Git) } 
+ WITH code_file
+     ,coalesce(code_file.absoluteFileName, code_file.fileName) AS codeFileName
+MATCH (git_file:Git&File)
+// Use repository if available to overcome ambiguity in multi source analysis
+OPTIONAL MATCH (git_repository:Git&Repository)-[:HAS_FILE]->(git_file)
  WITH *
+     ,git_repository
      ,git_file
-     ,coalesce(git_file.fileName, git_file.relativePath) AS gitFileName
+     ,coalesce(coalesce(git_repository.fileName + '/', '')  + git_file.fileName
+              ,coalesce(git_repository.name + '/', '')      + git_file.relativePath
+      ) AS gitFileName
 WHERE codeFileName      ENDS WITH gitFileName
 MERGE (git_file)-[:RESOLVES_TO]->(code_file)
   SET git_file.resolved = true
 RETURN count(DISTINCT codeFileName)  AS numberOfCodeFiles
-      ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
+      ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
+// RETURN codeFileName, gitFileName
+//       ,git_file.fileName, git_file.relativePath
+//       ,git_repository.fileName , code_file.absoluteFileName
+//       ,git_repository.name, code_file.fileName
+// LIMIT 20

cypher/GitLog/Delete_plain_git_directory_file_nodes.cypher

@@ -0,0 +1,14 @@
+// Delete plain file nodes in "/.git" directory
+
+MATCH (git_metadata_file:File)<-[:CONTAINS*]-(git_directory:Directory)
+WHERE git_directory.fileName ENDS WITH '/.git'
+ WITH git_directory.fileName AS gitDirectory
+     ,count(DISTINCT git_metadata_file.fileName)          AS numberOfFiles
+     ,collect(DISTINCT git_metadata_file.fileName)[0..4]  AS fileExamples
+     ,collect(DISTINCT git_metadata_file)                 AS git_metadata_files
+UNWIND git_metadata_files AS git_metadata_file
+  CALL { WITH git_metadata_file
+         DETACH DELETE git_metadata_file
+       } IN TRANSACTIONS OF 1000 ROWS
+RETURN DISTINCT gitDirectory, numberOfFiles, fileExamples
+ORDER BY numberOfFiles DESC

cypher/GitLog/Import_aggregated_git_log_csv_data.cypher

@@ -9,7 +9,7 @@ CALL { WITH row
         month:   toInteger(row.month),
         commits: toInteger(row.commits)
     })
-    MERGE (git_file:Git:Log:File {fileName: row.filename})
+    MERGE (git_file:Git:Log:File {fileName: row.filename, repositoryPath: $git_repository_absolute_directory_name})
     MERGE (git_author)-[:AUTHORED]->(git_change_span)
     MERGE (git_change_span)-[:CONTAINS_CHANGED]->(git_file)
     MERGE (git_repository)-[:HAS_CHANGE_SPAN]->(git_change_span)

cypher/GitLog/Import_git_log_csv_data.cypher

@@ -12,7 +12,7 @@ CALL { WITH row
         timestamp:      datetime(row.timestamp),
         timestamp_unix: toInteger(row.timestamp_unix)
     })
-    MERGE (git_file:Git:Log:File {fileName: row.filename})
+    MERGE (git_file:Git:Log:File {fileName: row.filename, repositoryPath: $git_repository_absolute_directory_name})
     MERGE (git_author)-[:AUTHORED]->(git_commit)
     MERGE (git_commit)-[:CONTAINS_CHANGED]->(git_file)
     MERGE (git_repository)-[:HAS_COMMIT]->(git_commit)

cypher/GitLog/List_git_files_by_resolved_label_and_extension.cypher

@@ -0,0 +1,14 @@
+// List resolved and unresolved git files by their extension
+
+MATCH (git_file:Git&File)
+OPTIONAL MATCH (git_file)-[:RESOLVES_TO]->(code_file:File&!Git)
+  WITH git_file
+      ,code_file
+      ,(code_file IS NOT NULL)                                      AS resolved
+      ,coalesce(git_file.fileName, git_file.relativePath)           AS gitFileName
+RETURN  resolved
+       ,reverse(split(split(reverse(gitFileName), '/')[0], '.')[0]) AS extension
+       ,count(DISTINCT git_file)                                    AS gitFileCount
+       ,coalesce(labels(code_file), labels(git_file))               AS fileLabels
+       ,collect(DISTINCT gitFileName)[0..9]                         AS gitFileExamples
+ORDER BY resolved ASC, gitFileCount DESC, extension ASC

cypher/GitLog/Verify_code_to_git_file_unambiguous.cypher

@@ -0,0 +1,16 @@
+// Verify that code to git file relationships aren't ambiguous
+
+MATCH (git_file:Git&File)-[:RESOLVES_TO]->(code_file:File&!Git)
+ WITH code_file
+     ,count(*)                    AS numberOfResolvedGitFiles
+     ,collect(DISTINCT git_file)  AS git_files
+WHERE numberOfResolvedGitFiles > 1
+UNWIND git_files AS git_file
+OPTIONAL MATCH (git_file)<-[:HAS_FILE]-(git_repository:Git&Repository)
+RETURN numberOfResolvedGitFiles
+      ,collect(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName))[0..4] AS codeFileExamples
+      ,collect(DISTINCT coalesce(git_file.fileName, git_file.relativePath))[0..4]       AS gitFileExamples
+      ,collect(DISTINCT git_repository.name)[0..4]                                      AS gitRepositoryExamples
+      //,collect(git_repository)[0..9]
+      //,collect(git_file)[0..9]
+      //,collect(code_file)[0..9]