Merge pull request #209 from JohT/feature/detect-and-mark-test-modules-for-typescript

JohT · web-flow · commit b2e54806bb79 · 2024-08-24T23:00:48.000+02:00
Detect, mark and filter test related modules in Typescript analysis
diff --git a/cypher/Dependencies_Projection/Dependencies_0_Prepare_Projection.cypher b/cypher/Dependencies_Projection/Dependencies_0_Prepare_Projection.cypher
@@ -0,0 +1,8 @@
+// Prepare projection by e.g. filling in default values for missing properties
+
+ MATCH (source)-[DEPENDS_ON]->(target)
+ WHERE $dependencies_projection_node IN labels(source)
+   AND $dependencies_projection_node IN labels(target)
+   SET source.testMarkerInteger = coalesce(source.testMarkerInteger, 0)
+      ,target.testMarkerInteger = coalesce(target.testMarkerInteger, 0)
+RETURN count(*) AS numberOfDependencies
diff --git a/cypher/Dependencies_Projection/Dependencies_3_Create_Projection.cypher b/cypher/Dependencies_Projection/Dependencies_3_Create_Projection.cypher
@@ -5,7 +5,8 @@
     $dependencies_projection_node, 
       'DEPENDS_ON', {
         relationshipProperties: [$dependencies_projection_weight_property],
-        nodeProperties: ['incomingDependencies', 'outgoingDependencies']
-  })
+        nodeProperties: ['incomingDependencies', 'outgoingDependencies', 'testMarkerInteger']
+      }
+  )
  YIELD graphName, nodeCount, relationshipCount
 RETURN graphName, nodeCount, relationshipCount
diff --git a/cypher/Dependencies_Projection/Dependencies_3b_Create_Multi_Relationship_Projection.cypher b/cypher/Dependencies_Projection/Dependencies_3b_Create_Multi_Relationship_Projection.cypher
@@ -16,7 +16,7 @@
             defaultValue: 1.0
           }
         },
-        nodeProperties: ['incomingDependencies', 'outgoingDependencies']
+        nodeProperties: ['incomingDependencies', 'outgoingDependencies', 'testMarkerInteger']
     }
   )
  YIELD graphName, nodeCount, relationshipCount
diff --git a/cypher/Dependencies_Projection/Dependencies_4_Create_Undirected_Projection.cypher b/cypher/Dependencies_Projection/Dependencies_4_Create_Undirected_Projection.cypher
@@ -10,7 +10,7 @@ CALL gds.graph.project(
     },
     {
         relationshipProperties: [$dependencies_projection_weight_property],
-        nodeProperties: ['incomingDependencies', 'outgoingDependencies']
+        nodeProperties: ['incomingDependencies', 'outgoingDependencies', 'testMarkerInteger']
     }
 )
  YIELD graphName, nodeCount, relationshipCount
diff --git a/cypher/Dependencies_Projection/Dependencies_5_Create_Subgraph.cypher b/cypher/Dependencies_Projection/Dependencies_5_Create_Subgraph.cypher
@@ -3,7 +3,7 @@
 CALL gds.graph.filter(
   $dependencies_projection + '-cleaned',
   $dependencies_projection,
-  'n.outgoingDependencies > 0 OR n.incomingDependencies > 0',
+  'n.testMarkerInteger = 0 AND (n.outgoingDependencies > 0 OR n.incomingDependencies > 0)',
   '*'
 )
  YIELD graphName, fromGraphName, nodeCount, relationshipCount, nodeFilter
diff --git a/cypher/External_Dependencies/External_module_usage_overall_for_Typescript.cypher b/cypher/External_Dependencies/External_module_usage_overall_for_Typescript.cypher
@@ -1,6 +1,7 @@
 // External Typescript module usage overall
 
  MATCH (internalModule:TS:Module)-[:EXPORTS]->(internalElement:TS)
+ WHERE NOT internalModule:TestRelated
   WITH count(DISTINCT internalElement.globalFqn)  AS allInternalElements
       ,count(DISTINCT internalModule.globalFqn)   AS allModules
       ,collect(DISTINCT internalElement)          AS internalElementList
diff --git a/cypher/External_Dependencies/External_module_usage_per_internal_module_aggregated_for_Typescript.cypher b/cypher/External_Dependencies/External_module_usage_per_internal_module_aggregated_for_Typescript.cypher
@@ -1,7 +1,8 @@
-// External Typescript module usage per interal module aggregated
+// External Typescript module usage per internal module aggregated
 
 // Get the overall internal module statistics first
  MATCH (internalModule:TS:Module)-[:EXPORTS]->(internalElement:TS)
+ WHERE NOT internalModule:TestRelated
   WITH internalModule.name                       AS internalModuleName
       ,internalModule.communityLeidenId          AS leidenCommunityId
       ,count(DISTINCT internalElement.globalFqn) AS internalModuleElementsCount
diff --git a/cypher/External_Dependencies/External_module_usage_per_internal_module_distribution_for_Typescript.cypher b/cypher/External_Dependencies/External_module_usage_per_internal_module_distribution_for_Typescript.cypher
@@ -1,6 +1,7 @@
 // External Typescript module usage distribution for internal modules
 
  MATCH (internalModule:TS:Module)-[:EXPORTS]->(internalElement:TS)
+ WHERE NOT internalModule:TestRelated
   WITH internalModule.name                       AS internalModuleName
       ,count(DISTINCT internalElement.globalFqn) AS numberOfAllInternalElements
       ,collect(DISTINCT internalElement)         AS internalElementList
diff --git a/cypher/External_Dependencies/External_module_usage_per_internal_module_sorted_for_Typescript.cypher b/cypher/External_Dependencies/External_module_usage_per_internal_module_sorted_for_Typescript.cypher
@@ -1,6 +1,7 @@
 // External Typescript module usage per internal module sorted by external usage descending
 
  MATCH (internalModule:TS:Module)-[:EXPORTS]->(internalElement:TS)
+ WHERE NOT internalModule:TestRelated
  OPTIONAL MATCH (internalElement)-[:DEPENDS_ON]->(externalDeclaration:ExternalDeclaration)
           WHERE externalDeclaration.isExternalImport = true
  OPTIONAL MATCH (externalModule:ExternalModule)-[:EXPORTS]->(externalDeclaration)
diff --git a/cypher/External_Dependencies/External_module_usage_spread_for_Typescript.cypher b/cypher/External_Dependencies/External_module_usage_spread_for_Typescript.cypher
@@ -2,6 +2,7 @@
 
 // Get the overall internal modules statistics first
  MATCH (internalModule:TS:Module)-[:EXPORTS]->(internalElement:TS)
+ WHERE NOT internalModule:TestRelated
   WITH count(DISTINCT internalModule.globalFqn)                  AS internalModulesCountOverall
       ,count(DISTINCT internalElement.globalFqn)                 AS internalElementsCountOverall
       ,collect(DISTINCT internalElement)                         AS internalElementList
diff --git a/cypher/External_Dependencies/External_namespace_usage_overall_for_Typescript.cypher b/cypher/External_Dependencies/External_namespace_usage_overall_for_Typescript.cypher
@@ -1,6 +1,7 @@
 // External Typescript namespace usage overall
 
  MATCH (internalModule:TS:Module)-[:EXPORTS]->(internalElement:TS)
+ WHERE NOT internalModule:TestRelated
   WITH count(DISTINCT internalElement.globalFqn)  AS allInternalElements
       ,count(DISTINCT internalModule.globalFqn)   AS allModules
       ,collect(DISTINCT internalElement)          AS internalElementList
@@ -11,18 +12,18 @@ UNWIND internalElementList AS internalElement
  MATCH (externalModule:ExternalModule)-[:EXPORTS]->(externalDeclaration)
   WITH allInternalElements
       ,allModules
-      ,externalModule.namespace                         AS externalNamespaceName
-      ,count(DISTINCT internalModule.globalFqn)         AS numberOfExternalCallerModules
-      ,count(DISTINCT internalElement.globalFqn)        AS numberOfExternalCallerElements
-      ,count(externalDependency)                        AS numberOfExternalDeclarationCalls
-      ,sum(externalDependency.cardinality)              AS numberOfExternalDeclarationCallsWeighted
+      ,coalesce(nullif(externalModule.namespace, ''), 'no namespace') AS externalNamespaceName
+      ,count(DISTINCT internalModule.globalFqn)      AS numberOfExternalCallerModules
+      ,count(DISTINCT internalElement.globalFqn)     AS numberOfExternalCallerElements
+      ,count(externalDependency)                     AS numberOfExternalDeclarationCalls
+      ,sum(externalDependency.cardinality)           AS numberOfExternalDeclarationCallsWeighted
       ,collect('<' + internalElement.name 
              + '> of module <'
              + internalModule.name
              + '> imports <'  
              + externalDeclaration.name
              + '> from external namespace <'
-             + externalModule.namespace + '>')[0..4]    AS exampleStories
+             + externalModule.namespace + '>')[0..4] AS exampleStories
 RETURN externalNamespaceName
       ,numberOfExternalCallerModules
       ,numberOfExternalCallerElements
diff --git a/cypher/External_Dependencies/External_namespace_usage_per_internal_module_sorted_for_Typescript.cypher b/cypher/External_Dependencies/External_namespace_usage_per_internal_module_sorted_for_Typescript.cypher
@@ -1,6 +1,7 @@
 // External Typescript module usage per internal module sorted by external usage descending
 
  MATCH (internalModule:TS:Module)-[:EXPORTS]->(internalElement:TS)
+ WHERE NOT internalModule:TestRelated
  OPTIONAL MATCH (internalElement)-[:DEPENDS_ON]->(externalDeclaration:ExternalDeclaration)
           WHERE externalDeclaration.isExternalImport = true
  OPTIONAL MATCH (externalModule:ExternalModule)-[:EXPORTS]->(externalDeclaration)
@@ -21,7 +22,7 @@ UNWIND internalElementList AS internalElement
       ,internalModuleName
       ,internalElement.globalFqn     AS fullInternalElementName
       ,internalElement.name          AS internalElementName
-      ,externalModule.namespace      AS externalNamespaceName
+      ,coalesce(nullif(externalModule.namespace, ''), 'no namespace') AS externalNamespaceName
       ,externalDeclaration.name      AS externalDeclarationName
   WITH numberOfAllElementsInInternalModule
       ,numberOfAllExternalDeclarationsUsedInInternalModule
diff --git a/cypher/External_Dependencies/External_namespace_usage_spread_for_Typescript.cypher b/cypher/External_Dependencies/External_namespace_usage_spread_for_Typescript.cypher
@@ -2,6 +2,7 @@
 
 // Get the overall internal modules statistics first
  MATCH (internalModule:TS:Module)-[:EXPORTS]->(internalElement:TS)
+ WHERE NOT internalModule:TestRelated
   WITH count(DISTINCT internalModule.globalFqn)                  AS internalModulesCountOverall
       ,count(DISTINCT internalElement.globalFqn)                 AS internalElementsCountOverall
       ,collect(DISTINCT internalElement)                         AS internalElementList
@@ -14,7 +15,7 @@ UNWIND internalElementList AS internalElement
  MATCH (externalModule:TS:ExternalModule)-[:EXPORTS]->(externalDeclaration)
   WITH internalModulesCountOverall
       ,internalElementsCountOverall
-      ,externalModule.namespace                               AS externalModuleNamespace
+      ,coalesce(nullif(externalModule.namespace, ''), 'no namespace') AS externalModuleNamespace
       ,coalesce(nullIf(internalModule.namespace, '') + '/' + internalModule.name, internalModule.name) AS internalModuleName
       
       // Gathering counts for every internal element and the external module it uses
diff --git a/cypher/Internal_Dependencies/How_many_elements_compared_to_all_existing_are_used_by_dependent_modules_for_Typescript.cypher b/cypher/Internal_Dependencies/How_many_elements_compared_to_all_existing_are_used_by_dependent_modules_for_Typescript.cypher
@@ -1,8 +1,8 @@
 // How many elements compared to all existing are used by dependent Typescript modules?
 
-MATCH (sourceModule:TS&Module)-[:EXPORTS]-(sourceElement:TS)
+MATCH (sourceModule:TS&Module&!TestRelated)-[:EXPORTS]-(sourceElement:TS)
 MATCH (sourceElement)-[:DEPENDS_ON]->(dependentElement:TS&!Module&!ExternalModule)
-MATCH (dependentModule:TS&Module)-[:EXPORTS]->(dependentElement)
+MATCH (dependentModule:TS&Module&!TestRelated)-[:EXPORTS]->(dependentElement)
 WHERE sourceModule <> dependentModule
 MATCH (dependentModule)-[:EXPORTS]->(dependentModuleElement:TS)
   WITH sourceModule.name                                  AS sourceModuleName
diff --git a/cypher/Internal_Dependencies/List_elements_that_are_used_by_many_different_modules_for_Typescript.cypher b/cypher/Internal_Dependencies/List_elements_that_are_used_by_many_different_modules_for_Typescript.cypher
@@ -1,8 +1,8 @@
 // List elements that are used by many different modules
 
-MATCH (sourceModule:TS&Module)-[:EXPORTS]->(sourceElement:TS)
+MATCH (sourceModule:TS&Module&!TestRelated)-[:EXPORTS]->(sourceElement:TS)
 MATCH (sourceElement)-[:DEPENDS_ON]->(dependentElement:TS&!Module&!ExternalModule)
-MATCH (dependentModule:TS&Module)-[:EXPORTS]->(dependentElement)
+MATCH (dependentModule:TS&Module&!TestRelated)-[:EXPORTS]->(dependentElement)
 WHERE sourceModule <> dependentModule
  WITH dependentElement
      ,labels(dependentElement)               AS dependentElementLabels
diff --git a/cypher/Typescript_Enrichment/Add_module_properties.cypher b/cypher/Typescript_Enrichment/Add_module_properties.cypher
@@ -7,7 +7,7 @@ OPTIONAL MATCH (class:TS:Class)-[:DECLARES]->(ts)
       ,reverse(split(reverse(replace(split(ts.globalFqn, '".')[0],'"', '')), '/')[0])       AS moduleName
       ,replace(split(split(ts.globalFqn, '/index')[0], '.default')[0],'"', '')              AS modulePathNameWithoutIndexAndDefault
       ,nullif(split(ts.globalFqn, '".')[1], 'default')                                      AS symbolName
-      ,split(nullif(reverse(split(reverse(ts.globalFqn), '@')[0]), ts.globalFqn), '/')[0]   AS namespaceName
+      ,split(split(reverse(split(reverse(ts.globalFqn), reverse('/node_modules/'))[0]),'@')[1], '/')[0] AS namespaceName
       ,(ts.globalFqn contains '/node_modules/')                                             AS isNodeModule
       ,((NOT ts.globalFqn STARTS WITH '/') AND size(split(ts.globalFqn, '/')) < 3)          AS isUnresolvedImport
       ,reverse(split(reverse(class.globalFqn), '.')[0])                                     AS optionalClassName
@@ -26,4 +26,15 @@ OPTIONAL MATCH (class:TS:Class)-[:DECLARES]->(ts)
        ,ts.isNodeModule       = isNodeModule
        ,ts.isUnresolvedImport = isUnresolvedImport
        ,ts.isExternalImport   = isNodeModule OR isUnresolvedImport
-RETURN count(*) AS updatedModules
+RETURN count(*) AS updatedModules
+// For debugging
+// RETURN namespaceNameWithAtPrefixed         AS namespace
+//        ,modulePathName                     AS module            
+//        ,moduleName                         AS moduleName        
+//        ,coalesce(symbolNameWithoutClassName, indexAndExtensionOmittedName) AS name              
+//        ,moduleNameExtensionExtended        AS extensionExtended 
+//        ,moduleNameExtension                AS extension         
+//        ,isNodeModule                       AS isNodeModule      
+//        ,isUnresolvedImport                 AS isUnresolvedImport
+//        ,isNodeModule OR isUnresolvedImport AS isExternalImport  
+// LIMIT 50
diff --git a/cypher/Typescript_Enrichment/Mark_test_modules.cypher b/cypher/Typescript_Enrichment/Mark_test_modules.cypher
@@ -0,0 +1,37 @@
+// Add "Test" label for modules that contain tests and test-related implementations
+
+MATCH (m:Module)
+ WITH m
+     ,(m.globalFqn contains '/__tests__/') OR        
+      (m.globalFqn contains '/test/')      OR
+      (m.globalFqn contains '/tests/')                  AS isInTestFolder
+     ,(m.globalFqn contains '/__mocks__/') OR        
+      (m.globalFqn contains '/mock/')      OR      
+      (m.globalFqn contains '/mocks/')                  AS isInMockFolder
+     ,(m.globalFqn ENDS WITH '.test.' + m.extension) OR
+      (m.globalFqn ENDS WITH '/test.' + m.extension) OR
+      (m.globalFqn ENDS WITH 'Test.'  + m.extension)    AS isTestExtension
+     ,(m.globalFqn ENDS WITH '.spec.' + m.extension) OR
+      (m.globalFqn ENDS WITH '/spec.' + m.extension) OR
+      (m.globalFqn ENDS WITH 'Spec.'  + m.extension)    AS isSpecificationExtension
+    //,(m.globalFqn contains 'test')                    AS isTestMentioned // for debugging
+  WITH *
+      ,(isTestExtension OR isSpecificationExtension) AS isTest
+      ,isInMockFolder                                AS isMock
+      ,(isInTestFolder AND NOT isInMockFolder AND NOT isTestExtension AND NOT isSpecificationExtension) AS isTestEnvironment
+  WITH *
+      ,CASE WHEN isTest            THEN [1] ELSE [] END AS nonEmptyIfTest
+      ,CASE WHEN isMock            THEN [1] ELSE [] END AS nonEmptyIfMock
+      ,CASE WHEN isTestEnvironment THEN [1] ELSE [] END AS nonEmptyIfTestEnvironment
+// Set labels "Test", "Mock", "TestEnvironment" and combining all "TestRelated" 
+// Additionally, set the property "testMarkerInteger" (default=0) to filter projections.
+// Numeric properties are required by projections using Neo4j's graph data science library.
+    SET m.testMarkerInteger = 0
+FOREACH (x in nonEmptyIfTest            | SET m:TestRelated:Test,            m.testMarkerInteger = 1)
+FOREACH (x in nonEmptyIfMock            | SET m:TestRelated:Mock,            m.testMarkerInteger = 2)
+FOREACH (x in nonEmptyIfTestEnvironment | SET m:TestRelated:TestEnvironment, m.testMarkerInteger = 3)
+RETURN isTest, isMock, isTestEnvironment
+      ,isInTestFolder, isInMockFolder, isTestExtension, isSpecificationExtension
+    //,isTestMentioned // for debugging
+     ,count(DISTINCT m.globalFqn)          AS numberOfModules
+     ,collect(DISTINCT m.globalFqn)[0..4]  AS examples 
diff --git a/scripts/prepareAnalysis.sh b/scripts/prepareAnalysis.sh
@@ -55,6 +55,7 @@ execute_cypher "${CYPHER_DIR}/Create_Typescript_index_for_name.cypher"
 # Preparation - Enrich Graph for Typescript by adding "module" and "name" properties
 execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Index_module_name.cypher"
 execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Add_module_properties.cypher"
+execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Mark_test_modules.cypher"
 
 # Preparation - Enrich Graph for Typescript by adding a name properties
 execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Add_name_to_property_on_projects.cypher"
diff --git a/scripts/projectionFunctions.sh b/scripts/projectionFunctions.sh
@@ -137,6 +137,7 @@ createDirectedDependencyProjection() {
         logNoDataForProjection "${@}"
         return 1
     fi
+    execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_0_Prepare_Projection.cypher" "${@}"
     execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_1_Delete_Projection.cypher" "${@}" >/dev/null
     execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_2_Delete_Subgraph.cypher" "${@}" >/dev/null
     execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_3_Create_Projection.cypher" "${@}"
@@ -175,6 +176,7 @@ createUndirectedDependencyProjection() {
     if [ "${projectionCheckResult}" -lt 1 ]; then
         return 1
     fi
+    execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_0_Prepare_Projection.cypher" "${@}"
     execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_1_Delete_Projection.cypher" "${@}" >/dev/null
     execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_2_Delete_Subgraph.cypher" "${@}" >/dev/null
     execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_4_Create_Undirected_Projection.cypher" "${@}"
diff --git a/scripts/reports/OverviewJupyter.sh b/scripts/reports/OverviewJupyter.sh
diff --git a/scripts/reports/WordcloudJupyter.sh b/scripts/reports/WordcloudJupyter.sh
diff --git a/scripts/resetAndScan.sh b/scripts/resetAndScan.sh
@@ -85,5 +85,5 @@ echo "resetAndScan: Analyzing using jQAssistant CLI version ${JQASSISTANT_CLI_VE
 
 "${JQASSISTANT_BIN}"/jqassistant.sh analyze
 
-# Scan all git repositories within the "source" (default) folder, scan their git log (history) and import it.
-source "${SCRIPTS_DIR}/importGit.sh"
+# Scan all git repositories within the "source" (default) folder and import their git log (history) if configured.
+time source "${SCRIPTS_DIR}/importGit.sh"

Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@`
`16`	`16`	`defaultValue: 1.0`
`17`	`17`	`}`
`18`	`18`	`},`
`19`		`- nodeProperties: ['incomingDependencies', 'outgoingDependencies']`
	`19`	`+ nodeProperties: ['incomingDependencies', 'outgoingDependencies', 'testMarkerInteger']`
`20`	`20`	`}`
`21`	`21`	`)`
`22`	`22`	`YIELD graphName, nodeCount, relationshipCount`
Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@ CALL gds.graph.project(`
`10`	`10`	`},`
`11`	`11`	`{`
`12`	`12`	`relationshipProperties: [$dependencies_projection_weight_property],`
`13`		`- nodeProperties: ['incomingDependencies', 'outgoingDependencies']`
	`13`	`+ nodeProperties: ['incomingDependencies', 'outgoingDependencies', 'testMarkerInteger']`
`14`	`14`	`}`
`15`	`15`	`)`
`16`	`16`	`YIELD graphName, nodeCount, relationshipCount`
Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@`
`3`	`3`	`CALL gds.graph.filter(`
`4`	`4`	`$dependencies_projection + '-cleaned',`
`5`	`5`	`$dependencies_projection,`
`6`		`- 'n.outgoingDependencies > 0 OR n.incomingDependencies > 0',`
	`6`	`+ 'n.testMarkerInteger = 0 AND (n.outgoingDependencies > 0 OR n.incomingDependencies > 0)',`
`7`	`7`	`'*'`
`8`	`8`	`)`
`9`	`9`	`YIELD graphName, fromGraphName, nodeCount, relationshipCount, nodeFilter`