Tune cypher queries for external dependencies

JohT · JohT · commit 40f8afa856aa · 2023-07-29T17:54:44.000+02:00
diff --git a/cypher/CYPHER.md b/cypher/CYPHER.md
@@ -67,18 +67,29 @@ Script | Directory | Description
 | [Community_Detection_for_Types_6_Leiden_Delete_Labels.cypher](./Community_Detection_for_Types/Community_Detection_for_Types_6_Leiden_Delete_Labels.cypher) | Community_Detection_for_Types | Community Detection for Types 6 Leiden Delete Labels |
 | [Community_Detection_for_Types_7_Add_LeidenTypeCommunity_Id_label_to_types.cypher](./Community_Detection_for_Types/Community_Detection_for_Types_7_Add_LeidenTypeCommunity_Id_label_to_types.cypher) | Community_Detection_for_Types | Community Detection for Types 7 Add LeidenTypeCommunity+Id label to types with more than one member |
 | [Which_type_community_spans_several_artifacts_and_how_are_the_types_distributed.cypher](./Community_Detection_for_Types/Which_type_community_spans_several_artifacts_and_how_are_the_types_distributed.cypher) | Community_Detection_for_Types | Which type community spans several artifacts and how are the types distributed? |
+| [Count_nodes_and_relationships.cypher](./Count_nodes_and_relationships.cypher) |  | Count nodes and relationships |
+| [Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_ARTIFACT.cypher](./Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_ARTIFACT.cypher) |  | Create a DEPENDS_ON relationship for every DEPENDS_ON_ARTIFACT |
+| [Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_PACKAGE.cypher](./Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_PACKAGE.cypher) |  | Create a DEPENDS_ON relationship for every DEPENDS_ON_PACKAGE |
+| [Create_index_for_full_qualified_type_name.cypher](./Create_index_for_full_qualified_type_name.cypher) |  | Create index for the full qualified type name |
 | [Cyclic_Dependencies.cypher](./Cyclic_Dependencies/Cyclic_Dependencies.cypher) | Cyclic_Dependencies | Cyclic Dependencies |
 | [Cyclic_Dependencies_Concatenated.cypher](./Cyclic_Dependencies/Cyclic_Dependencies_Concatenated.cypher) | Cyclic_Dependencies | Cyclic Dependencies Concatenated |
 | [Cyclic_Dependencies_as_List.cypher](./Cyclic_Dependencies/Cyclic_Dependencies_as_List.cypher) | Cyclic_Dependencies | Cyclic Dependencies as List |
 | [Cyclic_Dependencies_as_unwinded_List.cypher](./Cyclic_Dependencies/Cyclic_Dependencies_as_unwinded_List.cypher) | Cyclic_Dependencies | Cyclic Dependencies as unwinded List |
+| [Cyclic_Dependencies_between_Artrifacts_as_unwinded_List.cypher](./Cyclic_Dependencies/Cyclic_Dependencies_between_Artrifacts_as_unwinded_List.cypher) | Cyclic_Dependencies | Cyclic Dependencies between Artifacts as unwinded List |
 | [Export_the_whole_database_as_CSV.cypher](./Export_the_whole_database_as_CSV.cypher) |  | Export the whole database as CSV |
-| [External_package_usage_overall.cypher](./External_Dependencies/External_package_usage_overall.cypher) | External_Dependencies | External package usage overall |
-| [External_package_usage_per_artifact.cypher](./External_Dependencies/External_package_usage_per_artifact.cypher) | External_Dependencies | External package usage per artifact |
+| [External_package_usage_overall.cypher](./External_Dependencies/External_package_usage_overall.cypher) | External_Dependencies | External package usage overall tuned |
+| [External_package_usage_per_artifact.cypher](./External_Dependencies/External_package_usage_per_artifact.cypher) | External_Dependencies | External package usage per artifact tuned |
 | [External_package_usage_per_artifact_and_package.cypher](./External_Dependencies/External_package_usage_per_artifact_and_package.cypher) | External_Dependencies | External package usage per artifact and package |
-| [External_package_usage_per_type.cypher](./External_Dependencies/External_package_usage_per_type.cypher) | External_Dependencies | External package usage per type |
+| [External_package_usage_per_artifact_and_package_tuned.cypher](./External_Dependencies/External_package_usage_per_artifact_and_package_tuned.cypher) | External_Dependencies | External package usage per artifact and package tuned |
+| [External_package_usage_per_artifact_and_package_without_annotations.cypher](./External_Dependencies/External_package_usage_per_artifact_and_package_without_annotations.cypher) | External_Dependencies | External package usage per artifact and package without annotations Note: The exists operation for "isAnnotation" is inefficient for large graphs. |
+| [External_package_usage_per_type.cypher](./External_Dependencies/External_package_usage_per_type.cypher) | External_Dependencies | External package usage per type tuned |
 | [External_package_usage_per_type_distribution.cypher](./External_Dependencies/External_package_usage_per_type_distribution.cypher) | External_Dependencies | External package usage per type distribution |
+| [External_package_usage_per_type_distribution_without_annotations.cypher](./External_Dependencies/External_package_usage_per_type_distribution_without_annotations.cypher) | External_Dependencies | External package usage per type distribution without annotations |
 | [External_types_per_artifact_using_requires.cypher](./External_Dependencies/External_types_per_artifact_using_requires.cypher) | External_Dependencies | External types per artifact using requires |
+| [Label_external_types_and_annotations.cypher](./External_Dependencies/Label_external_types_and_annotations.cypher) | External_Dependencies | Label external types and annotations |
+| [List_external_types_used.cypher](./External_Dependencies/List_external_types_used.cypher) | External_Dependencies | List external types used |
 | [Maven_POMs_and_their_declared_dependencies.cypher](./External_Dependencies/Maven_POMs_and_their_declared_dependencies.cypher) | External_Dependencies | Maven POMs and their declared dependencies |
+| [Remove_external_type_and_annotation_labels.cypher](./External_Dependencies/Remove_external_type_and_annotation_labels.cypher) | External_Dependencies | Remove external type and annotation labels |
 | [Extract_Custom_Manifest_Entries.cypher](./Extract_Custom_Manifest_Entries.cypher) |  | Extract Custom Manifest Entries |
 | [Get_Awesome_Procedures_On_Cypher_APOC_Version.cypher](./Get_Awesome_Procedures_On_Cypher_APOC_Version.cypher) |  | Get Awesome Procedures On Cypher APOC Version |
 | [Get_Graph_Data_Science_Library_Version.cypher](./Get_Graph_Data_Science_Library_Version.cypher) |  | Get Graph Data Science Library Version |
diff --git a/cypher/External_Dependencies/External_package_usage_overall.cypher b/cypher/External_Dependencies/External_package_usage_overall.cypher
@@ -3,27 +3,15 @@
  MATCH (type:Type)
   WITH count(type) as allTypes, collect(type) as typeList
 UNWIND typeList AS type
- MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:Type)
+ MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
   WITH allTypes
       ,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
-      ,externalType.name                              AS externalTypeName
-      ,externalDependency
-      ,(NOT externalType.fqn CONTAINS '.')            AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')         AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type)) AS isAlsoInternalType
-      ,(externalType.byteCodeVersion IS NULL)         AS isExternalType
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
-   AND isExternalType     = true
-  WITH allTypes
-      ,externalPackageName
-      ,count(externalDependency)          AS numberOfExternalTypeCaller
-      ,sum(externalDependency.weight)     AS numberOfExternalTypeCalls
-      ,collect(DISTINCT externalTypeName) AS externalTypeNames
+      ,count(externalDependency)                              AS numberOfExternalTypeCaller
+      ,sum(externalDependency.weight)                         AS numberOfExternalTypeCalls
+      ,collect(DISTINCT externalType.name)                    AS externalTypeNames
 RETURN externalPackageName
       ,numberOfExternalTypeCaller
       ,numberOfExternalTypeCalls
       ,allTypes
       ,externalTypeNames
- ORDER BY numberOfExternalTypeCaller DESC, externalTypeNames ASC
+ ORDER BY numberOfExternalTypeCaller DESC, externalPackageName ASC
diff --git a/cypher/External_Dependencies/External_package_usage_per_artifact.cypher b/cypher/External_Dependencies/External_package_usage_per_artifact.cypher
@@ -1,23 +1,18 @@
-// External package usage per artifact 
+// External package usage per artifact
 
- MATCH (artifact:Artifact)-[:CONTAINS]->(type:Type)
-  WITH artifact, count(type) as numberOfTypesInArtifact, collect(type) as typeList
-UNWIND typeList as type
- MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:Type)
- WHERE externalType.byteCodeVersion IS NULL
+ MATCH (artifact:Artifact:Archive)-[:CONTAINS]->(type:Type)
+  WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
+      ,count(type)   AS numberOfTypesInArtifact
+      ,collect(type) AS typeList
+UNWIND typeList AS type
+ MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
   WITH numberOfTypesInArtifact
       ,externalDependency
-      ,replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
+      ,artifactName
       ,type.fqn     AS fullTypeName
       ,type.name    AS typeName
       ,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
       ,externalType.name                              AS externalTypeName
-      ,(NOT externalType.fqn CONTAINS '.')            AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')         AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type)) AS isAlsoInternalType
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
   WITH numberOfTypesInArtifact
       ,artifactName
       ,externalPackageName
@@ -30,4 +25,4 @@ RETURN artifactName
       ,numberOfExternalTypeCalls
       ,numberOfTypesInArtifact
       ,externalTypeNames
-ORDER BY artifactName ASC, numberOfExternalTypeCaller DESC
+ORDER BY artifactName ASC, numberOfExternalTypeCaller DESC, externalPackageName ASC
diff --git a/cypher/External_Dependencies/External_package_usage_per_artifact_and_package.cypher b/cypher/External_Dependencies/External_package_usage_per_artifact_and_package.cypher
@@ -1,28 +1,24 @@
-// External package usage per artifact and package 
+// External package usage per artifact and package
 
  MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
  MATCH (package)-[:CONTAINS]->(type:Type)
-  WITH artifact, package, count(type) AS numberOfTypesInPackage, collect(type) as typeList
+  WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
+      ,package.fqn   AS fullPackageName
+      ,package.name  AS packageName
+      ,count(type)   AS numberOfTypesInPackage
+      ,collect(type) AS typeList
 UNWIND typeList AS type
- MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:Type)
- WHERE externalType.byteCodeVersion IS NULL
+ MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
+ WHERE NOT externalType:ExternalAnnotation
   WITH numberOfTypesInPackage
       ,externalDependency
-      ,replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
-      ,package.fqn  AS fullPackageName
-      ,package.name AS packageName
+      ,artifactName
+      ,fullPackageName
+      ,packageName
       ,type.fqn     AS fullTypeName
       ,type.name    AS typeName
       ,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
       ,externalType.name                              AS externalTypeName
-      ,(NOT externalType.fqn CONTAINS '.')            AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')         AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type)) AS isAlsoInternalType
-      ,exists((externalType)<-[:OF_TYPE]-()<-[:ANNOTATED_BY]-()) AS isAnnotation
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
-   AND isAnnotation       = false
   WITH numberOfTypesInPackage
       ,artifactName
       ,fullPackageName
diff --git a/cypher/External_Dependencies/External_package_usage_per_artifact_and_package_with_annotations.cypher b/cypher/External_Dependencies/External_package_usage_per_artifact_and_package_with_annotations.cypher
@@ -0,0 +1,34 @@
+// External package usage per artifact and package with external annotations
+
+ MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
+ MATCH (package)-[:CONTAINS]->(type:Type)
+  WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
+      ,package.fqn   AS fullPackageName
+      ,package.name  AS packageName
+      ,count(type)   AS numberOfTypesInPackage
+      ,collect(type) AS typeList
+UNWIND typeList AS type
+ MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
+  WITH numberOfTypesInPackage
+      ,externalDependency
+      ,artifactName
+      ,fullPackageName
+      ,packageName
+      ,type.fqn     AS fullTypeName
+      ,type.name    AS typeName
+      ,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
+      ,externalType.name                              AS externalTypeName
+  WITH numberOfTypesInPackage
+      ,artifactName
+      ,fullPackageName
+      ,packageName
+      ,externalPackageName
+      ,count(externalDependency)          AS numberOfExternalTypeCaller
+      ,sum(externalDependency.weight)     AS numberOfExternalTypeCalls
+      ,collect(DISTINCT externalTypeName) AS externalTypeNames
+RETURN artifactName, fullPackageName
+      ,externalPackageName
+      ,numberOfExternalTypeCaller, numberOfExternalTypeCalls, numberOfTypesInPackage
+      ,externalTypeNames
+      ,packageName
+ORDER BY numberOfExternalTypeCaller DESC, artifactName ASC, fullPackageName ASC
diff --git a/cypher/External_Dependencies/External_package_usage_per_type.cypher b/cypher/External_Dependencies/External_package_usage_per_type.cypher
@@ -2,8 +2,7 @@
 
  MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
  MATCH (package)-[:CONTAINS]->(type:Type)
- MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:Type)
- WHERE externalType.byteCodeVersion IS NULL
+ MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
   WITH externalDependency
       ,replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
       ,package.fqn  AS fullPackageName
@@ -12,12 +11,6 @@
       ,type.name    AS typeName
       ,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
       ,externalType.name                              AS externalTypeName
-      ,(NOT externalType.fqn CONTAINS '.')            AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')         AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type)) AS isAlsoInternalType
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
   WITH artifactName
       ,fullPackageName
       ,packageName
diff --git a/cypher/External_Dependencies/External_package_usage_per_type_distribution.cypher b/cypher/External_Dependencies/External_package_usage_per_type_distribution.cypher
@@ -5,20 +5,12 @@
       ,count(type)                                               AS artifactTypes
       ,collect(type)                                             AS typeList
 UNWIND typeList AS type
- MATCH (type)-[:DEPENDS_ON]->(externalType:Type)
- WHERE externalType.byteCodeVersion IS NULL
+ MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
+ WHERE externalType:ExternalAnnotation
   WITH artifactName
       ,artifactTypes
       ,type.fqn                                                  AS fullTypeName
       ,replace(externalType.fqn, '.' + externalType.name, '')    AS externalPackageName
-      ,(NOT externalType.fqn CONTAINS '.')                       AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')                    AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type))            AS isAlsoInternalType
-      ,exists((externalType)<-[:OF_TYPE]-()<-[:ANNOTATED_BY]-()) AS isAnnotation
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
-   AND isAnnotation       = false
   WITH artifactName
       ,artifactTypes
       ,fullTypeName
diff --git a/cypher/External_Dependencies/External_package_usage_per_type_distribution_with_annotations.cypher b/cypher/External_Dependencies/External_package_usage_per_type_distribution_with_annotations.cypher
@@ -0,0 +1,28 @@
+// External package usage per type distribution with external annotations
+
+ MATCH (artifact:Artifact)-[:CONTAINS]->(type:Type)
+  WITH replace(last(split(artifact.fileName, '/')), '.jar', '')  AS artifactName
+      ,count(type)                                               AS artifactTypes
+      ,collect(type)                                             AS typeList
+UNWIND typeList AS type
+ MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
+  WITH artifactName
+      ,artifactTypes
+      ,type.fqn                                                  AS fullTypeName
+      ,replace(externalType.fqn, '.' + externalType.name, '')    AS externalPackageName
+  WITH artifactName
+      ,artifactTypes
+      ,fullTypeName
+      ,count(DISTINCT externalPackageName) AS numberOfExternalPackages    
+  WITH artifactName
+      ,artifactTypes
+      ,numberOfExternalPackages
+      ,count(DISTINCT fullTypeName)   AS numberOfTypes
+      ,COLLECT(DISTINCT fullTypeName) AS nameOfTypes
+RETURN artifactName
+      ,artifactTypes
+      ,numberOfExternalPackages
+      ,numberOfTypes
+      ,100.0 / artifactTypes * numberOfTypes AS numberOfTypesPercentage
+      ,nameOfTypes
+ORDER BY artifactName ASC, numberOfExternalPackages ASC
diff --git a/cypher/External_Dependencies/External_types_per_artifact_using_requires.cypher b/cypher/External_Dependencies/External_types_per_artifact_using_requires.cypher
@@ -1,11 +1,8 @@
 // External types per artifact using requires
 
-MATCH (artifact:Artifact)-[:REQUIRES]->(externalType:Type)
+MATCH (artifact:Artifact)-[:REQUIRES]->(externalType:ExternalType)
 MATCH (artifact)-[:CONTAINS]->(caller:Type)
 OPTIONAL MATCH (caller)-[callerDependency:DEPENDS_ON]->(externalType)
-WHERE NOT externalType.fqn STARTS WITH 'java.' // ignore 
-  AND externalType.fqn CONTAINS '.' // ignore primitives
-  AND NOT EXISTS((externalType)-[:RESOLVES_TO]->(:Type))
  WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
      ,replace(externalType.fqn, '.' + externalType.name, '') AS externalTypePackage
      ,COLLECT(DISTINCT externalType.name)   AS externalTypeNames
diff --git a/cypher/External_Dependencies/Label_external_types_and_annotations.cypher b/cypher/External_Dependencies/Label_external_types_and_annotations.cypher
@@ -0,0 +1,17 @@
+// Label external types and annotations
+
+ MATCH (externalType:Type)
+ WHERE externalType.byteCodeVersion IS NULL // byte code not available -> external dependency
+  WITH externalType
+      ,(NOT externalType.fqn CONTAINS '.')                       AS isPrimitiveType
+      ,(externalType.fqn STARTS WITH 'java.')                    AS isJavaType
+      ,exists((externalType)-[:RESOLVES_TO]->(:Type))            AS isAlsoInternalType
+      ,exists((externalType)<-[:OF_TYPE]-()<-[:ANNOTATED_BY]-()) AS isAnnotation
+ WHERE isPrimitiveType    = false
+   AND isJavaType         = false
+   AND isAlsoInternalType = false
+  WITH externalType
+      ,CASE WHEN isAnnotation THEN [1] ELSE [] END AS annotated
+FOREACH (x in annotated | SET externalType:ExternalAnnotation)
+    SET externalType:ExternalType 
+ RETURN labels(externalType), count(externalType) as numberOfExternalTypes
diff --git a/cypher/External_Dependencies/List_external_types_used.cypher b/cypher/External_Dependencies/List_external_types_used.cypher
@@ -0,0 +1,3 @@
+// List external types used
+
+MATCH (external:ExternalType) RETURN external.fqn
diff --git a/cypher/External_Dependencies/Remove_external_type_and_annotation_labels.cypher b/cypher/External_Dependencies/Remove_external_type_and_annotation_labels.cypher
@@ -0,0 +1,4 @@
+// Remove external type and annotation labels
+
+  MATCH (externalType:ExternalType)
+ REMOVE externalType:ExternalType:ExternalAnnotation
diff --git a/scripts/prepareAnalysis.sh b/scripts/prepareAnalysis.sh
@@ -27,6 +27,7 @@ source "${SCRIPTS_DIR}/executeQueryFunctions.sh"
 # Local Constants
 PACKAGE_WEIGHTS_CYPHER_DIR="$CYPHER_DIR/Package_Relationship_Weights"
 PACKAGE_METRICS_CYPHER_DIR="$CYPHER_DIR/Metrics"
+EXTERNAL_DEPENDENCIES_CYPHER_DIR="$CYPHER_DIR/External_Dependencies"
 
 # Preparation - Create indizes
 execute_cypher "${CYPHER_DIR}/Create_index_for_full_qualified_type_name.cypher" || exit 1
@@ -44,3 +45,9 @@ execute_cypher_expect_results "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight10Percent
 # Preparation - Add Package node properties "incomingDependencies" and "outgoingDependencies"
 execute_cypher_expect_results "${PACKAGE_METRICS_CYPHER_DIR}/Set_Incoming_Package_Dependencies.cypher" || exit 1
 execute_cypher_expect_results "${PACKAGE_METRICS_CYPHER_DIR}/Set_Outgoing_Package_Dependencies.cypher" || exit 1
+
+# Preparation - Label external types and annotations
+#               "external" means that there is no byte code available, not a primitive type and not a java type
+#               "annoatation" means that there is a ANNOTATED_BY to that external type
+execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/Remove_external_type_and_annotation_labels.cypher" || exit 1
+execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/Label_external_types_and_annotations.cypher" || exit 1
diff --git a/scripts/reports/ExternalDependenciesCsv.sh b/scripts/reports/ExternalDependenciesCsv.sh
