Tune cypher queries for external dependencies

JohT · JohT · commit 763800ddbd6b · 2023-07-29T11:29:08.000+02:00
diff --git a/cypher/CYPHER.md b/cypher/CYPHER.md
@@ -70,22 +70,26 @@ Script | Directory | Description
 | [Count_nodes_and_relationships.cypher](./Count_nodes_and_relationships.cypher) |  | Count nodes and relationships |
 | [Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_ARTIFACT.cypher](./Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_ARTIFACT.cypher) |  | Create a DEPENDS_ON relationship for every DEPENDS_ON_ARTIFACT |
 | [Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_PACKAGE.cypher](./Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_PACKAGE.cypher) |  | Create a DEPENDS_ON relationship for every DEPENDS_ON_PACKAGE |
-| [Create_index_for_full_qualified_type_name.cypher](./Create_index_for_full_qualified_type_name.cypher) |  | Create index for full qualified type name |
+| [Create_index_for_full_qualified_type_name.cypher](./Create_index_for_full_qualified_type_name.cypher) |  | Create index for the full qualified type name |
 | [Cyclic_Dependencies.cypher](./Cyclic_Dependencies/Cyclic_Dependencies.cypher) | Cyclic_Dependencies | Cyclic Dependencies |
 | [Cyclic_Dependencies_Concatenated.cypher](./Cyclic_Dependencies/Cyclic_Dependencies_Concatenated.cypher) | Cyclic_Dependencies | Cyclic Dependencies Concatenated |
 | [Cyclic_Dependencies_as_List.cypher](./Cyclic_Dependencies/Cyclic_Dependencies_as_List.cypher) | Cyclic_Dependencies | Cyclic Dependencies as List |
 | [Cyclic_Dependencies_as_unwinded_List.cypher](./Cyclic_Dependencies/Cyclic_Dependencies_as_unwinded_List.cypher) | Cyclic_Dependencies | Cyclic Dependencies as unwinded List |
 | [Cyclic_Dependencies_between_Artrifacts_as_unwinded_List.cypher](./Cyclic_Dependencies/Cyclic_Dependencies_between_Artrifacts_as_unwinded_List.cypher) | Cyclic_Dependencies | Cyclic Dependencies between Artifacts as unwinded List |
 | [Export_the_whole_database_as_CSV.cypher](./Export_the_whole_database_as_CSV.cypher) |  | Export the whole database as CSV |
-| [External_package_usage_overall.cypher](./External_Dependencies/External_package_usage_overall.cypher) | External_Dependencies | External package usage overall |
-| [External_package_usage_per_artifact.cypher](./External_Dependencies/External_package_usage_per_artifact.cypher) | External_Dependencies | External package usage per artifact |
+| [External_package_usage_overall.cypher](./External_Dependencies/External_package_usage_overall.cypher) | External_Dependencies | External package usage overall tuned |
+| [External_package_usage_per_artifact.cypher](./External_Dependencies/External_package_usage_per_artifact.cypher) | External_Dependencies | External package usage per artifact tuned |
 | [External_package_usage_per_artifact_and_package.cypher](./External_Dependencies/External_package_usage_per_artifact_and_package.cypher) | External_Dependencies | External package usage per artifact and package |
+| [External_package_usage_per_artifact_and_package_tuned.cypher](./External_Dependencies/External_package_usage_per_artifact_and_package_tuned.cypher) | External_Dependencies | External package usage per artifact and package tuned |
 | [External_package_usage_per_artifact_and_package_without_annotations.cypher](./External_Dependencies/External_package_usage_per_artifact_and_package_without_annotations.cypher) | External_Dependencies | External package usage per artifact and package without annotations Note: The exists operation for "isAnnotation" is inefficient for large graphs. |
-| [External_package_usage_per_type.cypher](./External_Dependencies/External_package_usage_per_type.cypher) | External_Dependencies | External package usage per type |
+| [External_package_usage_per_type.cypher](./External_Dependencies/External_package_usage_per_type.cypher) | External_Dependencies | External package usage per type tuned |
 | [External_package_usage_per_type_distribution.cypher](./External_Dependencies/External_package_usage_per_type_distribution.cypher) | External_Dependencies | External package usage per type distribution |
-| [External_package_usage_per_type_distribution_without_annotations.cypher](./External_Dependencies/External_package_usage_per_type_distribution_without_annotations.cypher) | External_Dependencies | External package usage per type distribution without annotations Note: The exists operation for "isAnnotation" is inefficient for large graphs. |
+| [External_package_usage_per_type_distribution_without_annotations.cypher](./External_Dependencies/External_package_usage_per_type_distribution_without_annotations.cypher) | External_Dependencies | External package usage per type distribution without annotations |
 | [External_types_per_artifact_using_requires.cypher](./External_Dependencies/External_types_per_artifact_using_requires.cypher) | External_Dependencies | External types per artifact using requires |
+| [Label_external_types_and_annotations.cypher](./External_Dependencies/Label_external_types_and_annotations.cypher) | External_Dependencies | Label external types and annotations |
+| [List_external_types_used.cypher](./External_Dependencies/List_external_types_used.cypher) | External_Dependencies | List external types used |
 | [Maven_POMs_and_their_declared_dependencies.cypher](./External_Dependencies/Maven_POMs_and_their_declared_dependencies.cypher) | External_Dependencies | Maven POMs and their declared dependencies |
+| [Remove_external_type_and_annotation_labels.cypher](./External_Dependencies/Remove_external_type_and_annotation_labels.cypher) | External_Dependencies | Remove external type and annotation labels |
 | [Extract_Custom_Manifest_Entries.cypher](./Extract_Custom_Manifest_Entries.cypher) |  | Extract Custom Manifest Entries |
 | [Get_Awesome_Procedures_On_Cypher_APOC_Version.cypher](./Get_Awesome_Procedures_On_Cypher_APOC_Version.cypher) |  | Get Awesome Procedures On Cypher APOC Version |
 | [Get_Graph_Data_Science_Library_Version.cypher](./Get_Graph_Data_Science_Library_Version.cypher) |  | Get Graph Data Science Library Version |
diff --git a/cypher/External_Dependencies/External_package_usage_overall.cypher b/cypher/External_Dependencies/External_package_usage_overall.cypher
@@ -1,26 +1,14 @@
-// External package usage overall
+// External package usage overall tuned
 
  MATCH (type:Type)
   WITH count(type) as allTypes, collect(type) as typeList
 UNWIND typeList AS type
- MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:Type)
+ MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
   WITH allTypes
       ,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
-      ,externalType.name                              AS externalTypeName
-      ,externalDependency
-      ,(NOT externalType.fqn CONTAINS '.')            AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')         AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type)) AS isAlsoInternalType
-      ,(externalType.byteCodeVersion IS NULL)         AS isExternalType
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
-   AND isExternalType     = true
-  WITH allTypes
-      ,externalPackageName
-      ,count(externalDependency)          AS numberOfExternalTypeCaller
-      ,sum(externalDependency.weight)     AS numberOfExternalTypeCalls
-      ,collect(DISTINCT externalTypeName) AS externalTypeNames
+      ,count(externalDependency)                              AS numberOfExternalTypeCaller
+      ,sum(externalDependency.weight)                         AS numberOfExternalTypeCalls
+      ,collect(DISTINCT externalType.name)                    AS externalTypeNames
 RETURN externalPackageName
       ,numberOfExternalTypeCaller
       ,numberOfExternalTypeCalls
diff --git a/cypher/External_Dependencies/External_package_usage_per_artifact.cypher b/cypher/External_Dependencies/External_package_usage_per_artifact.cypher
@@ -1,27 +1,22 @@
-// External package usage per artifact 
+// External package usage per artifact tuned
 
- MATCH (artifact:Artifact)-[:CONTAINS]->(type:Type)
-  WITH artifact, count(type) as numberOfTypesInArtifact, collect(type) as typeList
+ MATCH (artifact:Artifact:Archive)-[:CONTAINS]->(type:Type)
+  WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
+      ,count(type)                                              AS numberOfTypesInArtifact
+      ,collect(type)                                            AS typeList
 UNWIND typeList as type
- MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:Type)
- WHERE externalType.byteCodeVersion IS NULL
+ MATCH (externalType:ExternalType)<-[externalDependency:DEPENDS_ON]-(type)
   WITH numberOfTypesInArtifact
       ,externalDependency
-      ,replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
+      ,artifactName
       ,type.fqn     AS fullTypeName
       ,type.name    AS typeName
       ,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
-      ,externalType.name                              AS externalTypeName
-      ,(NOT externalType.fqn CONTAINS '.')            AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')         AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type)) AS isAlsoInternalType
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
+      ,externalType.name                                      AS externalTypeName
   WITH numberOfTypesInArtifact
       ,artifactName
       ,externalPackageName
-      ,count(externalDependency)          AS numberOfExternalTypeCaller
+      ,count(DISTINCT fullTypeName)       AS numberOfExternalTypeCaller
       ,sum(externalDependency.weight)     AS numberOfExternalTypeCalls
       ,collect(DISTINCT externalTypeName) AS externalTypeNames
 RETURN artifactName
diff --git a/cypher/External_Dependencies/External_package_usage_per_artifact_and_package_tuned.cypher b/cypher/External_Dependencies/External_package_usage_per_artifact_and_package_tuned.cypher
@@ -0,0 +1,36 @@
+// External package usage per artifact and package tuned
+
+ MATCH (artifact:Artifact:Archive)-[:CONTAINS]->(package:Package)
+ MATCH (package)-[:CONTAINS]->(type:Type)
+  WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
+      ,package.fqn                                              AS fullPackageName
+      ,package.name                                             AS packageName
+      ,count(type)                                              AS numberOfTypesInPackage
+      ,collect(type)                                            AS typeList
+UNWIND typeList AS type
+ MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
+ WHERE externalType.byteCodeVersion IS NULL
+   AND NOT externalType:ExternalAnnotation
+  WITH numberOfTypesInPackage
+      ,externalDependency
+      ,artifactName
+      ,fullPackageName
+      ,packageName
+      ,type.fqn     AS fullTypeName
+      ,type.name    AS typeName
+      ,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
+      ,externalType.name                                      AS externalTypeName
+  WITH numberOfTypesInPackage
+      ,artifactName
+      ,fullPackageName
+      ,packageName
+      ,externalPackageName
+      ,count(externalDependency)          AS numberOfExternalTypeCaller
+      ,sum(externalDependency.weight)     AS numberOfExternalTypeCalls
+      ,collect(DISTINCT externalTypeName) AS externalTypeNames
+RETURN artifactName, fullPackageName
+      ,externalPackageName
+      ,numberOfExternalTypeCaller, numberOfExternalTypeCalls, numberOfTypesInPackage
+      ,externalTypeNames
+      ,packageName
+ORDER BY numberOfExternalTypeCaller DESC, artifactName ASC, fullPackageName ASC
diff --git a/cypher/External_Dependencies/External_package_usage_per_type.cypher b/cypher/External_Dependencies/External_package_usage_per_type.cypher
@@ -1,9 +1,8 @@
-// External package usage per type
+// External package usage per type tuned
 
- MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
+ MATCH (artifact:Artifact:Archive)-[:CONTAINS]->(package:Package)
  MATCH (package)-[:CONTAINS]->(type:Type)
- MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:Type)
- WHERE externalType.byteCodeVersion IS NULL
+ MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
   WITH externalDependency
       ,replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
       ,package.fqn  AS fullPackageName
@@ -12,12 +11,6 @@
       ,type.name    AS typeName
       ,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
       ,externalType.name                              AS externalTypeName
-      ,(NOT externalType.fqn CONTAINS '.')            AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')         AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type)) AS isAlsoInternalType
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
   WITH artifactName
       ,fullPackageName
       ,packageName
@@ -30,8 +23,8 @@
       ,count  (DISTINCT externalPackageName + '.' + externalTypeName) AS numberOfExternalTypes
       ,collect(DISTINCT externalPackageName + '.' + externalTypeName) AS externalTypeNames
 RETURN artifactName, fullPackageName, typeName
-      ,numberOfExternalTypeCaller, numberOfExternalTypeCalls, numberOfExternalPackages, numberOfExternalTypes
+      ,numberOfExternalTypeCaller, numberOfExternalTypeCalls
       ,externalPackageNames, externalTypeNames
       ,packageName
       ,fullTypeName
-ORDER BY numberOfExternalPackages DESC, artifactName ASC, fullPackageName ASC
+ORDER BY artifactName ASC, fullPackageName ASC, numberOfExternalTypeCalls DESC
diff --git a/cypher/External_Dependencies/External_package_usage_per_type_distribution.cypher b/cypher/External_Dependencies/External_package_usage_per_type_distribution.cypher
@@ -1,24 +1,15 @@
 // External package usage per type distribution
 
- MATCH (artifact:Artifact)-[:CONTAINS]->(type:Type)
+ MATCH (artifact:Artifact:Archive)-[:CONTAINS]->(type:Type)
   WITH replace(last(split(artifact.fileName, '/')), '.jar', '')  AS artifactName
       ,count(type)                                               AS artifactTypes
       ,collect(type)                                             AS typeList
 UNWIND typeList AS type
- MATCH (type)-[:DEPENDS_ON]->(externalType:Type)
- WHERE externalType.byteCodeVersion IS NULL
+ MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
   WITH artifactName
       ,artifactTypes
       ,type.fqn                                                  AS fullTypeName
       ,replace(externalType.fqn, '.' + externalType.name, '')    AS externalPackageName
-      ,(NOT externalType.fqn CONTAINS '.')                       AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')                    AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type))            AS isAlsoInternalType
-//      ,exists((externalType)<-[:OF_TYPE]-()<-[:ANNOTATED_BY]-()) AS isAnnotation
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
-//   AND isAnnotation       = false
   WITH artifactName
       ,artifactTypes
       ,fullTypeName
@@ -32,6 +23,5 @@ RETURN artifactName
       ,artifactTypes
       ,numberOfExternalPackages
       ,numberOfTypes
-      ,100.0 / artifactTypes * numberOfTypes AS numberOfTypesPercentage
       ,nameOfTypes
 ORDER BY artifactName ASC, numberOfExternalPackages ASC
diff --git a/cypher/External_Dependencies/External_package_usage_per_type_distribution_without_annotations.cypher b/cypher/External_Dependencies/External_package_usage_per_type_distribution_without_annotations.cypher
@@ -1,25 +1,16 @@
 // External package usage per type distribution without annotations
-// Note: The exists operation for "isAnnotation" is inefficient for large graphs.  
 
- MATCH (artifact:Artifact)-[:CONTAINS]->(type:Type)
+ MATCH (artifact:Artifact:Archive)-[:CONTAINS]->(type:Type)
   WITH replace(last(split(artifact.fileName, '/')), '.jar', '')  AS artifactName
       ,count(type)                                               AS artifactTypes
       ,collect(type)                                             AS typeList
 UNWIND typeList AS type
- MATCH (type)-[:DEPENDS_ON]->(externalType:Type)
- WHERE externalType.byteCodeVersion IS NULL
+ MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
+ WHERE NOT externalType:ExternalAnnotation
   WITH artifactName
       ,artifactTypes
       ,type.fqn                                                  AS fullTypeName
       ,replace(externalType.fqn, '.' + externalType.name, '')    AS externalPackageName
-      ,(NOT externalType.fqn CONTAINS '.')                       AS isPrimitiveType
-      ,(externalType.fqn STARTS WITH 'java.')                    AS isJavaType
-      ,exists((externalType)-[:RESOLVES_TO]->(:Type))            AS isAlsoInternalType
-      ,exists((externalType)<-[:OF_TYPE]-()<-[:ANNOTATED_BY]-()) AS isAnnotation
- WHERE isPrimitiveType    = false
-   AND isJavaType         = false
-   AND isAlsoInternalType = false
-   AND isAnnotation       = false
   WITH artifactName
       ,artifactTypes
       ,fullTypeName
@@ -33,6 +24,5 @@ RETURN artifactName
       ,artifactTypes
       ,numberOfExternalPackages
       ,numberOfTypes
-      ,100.0 / artifactTypes * numberOfTypes AS numberOfTypesPercentage
       ,nameOfTypes
 ORDER BY artifactName ASC, numberOfExternalPackages ASC
diff --git a/cypher/External_Dependencies/Label_external_types_and_annotations.cypher b/cypher/External_Dependencies/Label_external_types_and_annotations.cypher
@@ -0,0 +1,17 @@
+// Label external types and annotations
+
+ MATCH (externalType:Type)
+ WHERE externalType.byteCodeVersion IS NULL // byte code not available -> external dependency
+  WITH externalType
+      ,(NOT externalType.fqn CONTAINS '.')                       AS isPrimitiveType
+      ,(externalType.fqn STARTS WITH 'java.')                    AS isJavaType
+      ,exists((externalType)-[:RESOLVES_TO]->(:Type))            AS isAlsoInternalType
+      ,exists((externalType)<-[:OF_TYPE]-()<-[:ANNOTATED_BY]-()) AS isAnnotation
+ WHERE isPrimitiveType    = false
+   AND isJavaType         = false
+   AND isAlsoInternalType = false
+  WITH externalType
+      ,CASE WHEN isAnnotation THEN [1] ELSE [] END AS annotated
+FOREACH (x in annotated | SET externalType:ExternalAnnotation)
+    SET externalType:ExternalType 
+ RETURN labels(externalType), count(externalType) as numberOfExternalTypes
diff --git a/cypher/External_Dependencies/List_external_types_used.cypher b/cypher/External_Dependencies/List_external_types_used.cypher
@@ -0,0 +1,3 @@
+// List external types used
+
+MATCH (external:ExternalType) RETURN external.fqn
diff --git a/cypher/External_Dependencies/Remove_external_type_and_annotation_labels.cypher b/cypher/External_Dependencies/Remove_external_type_and_annotation_labels.cypher
@@ -0,0 +1,4 @@
+// Remove external type and annotation labels
+
+  MATCH (externalType:ExternalType)
+ REMOVE externalType:ExternalType:ExternalAnnotation
diff --git a/scripts/prepareAnalysis.sh b/scripts/prepareAnalysis.sh
@@ -27,6 +27,7 @@ source "${SCRIPTS_DIR}/executeQueryFunctions.sh"
 # Local Constants
 PACKAGE_WEIGHTS_CYPHER_DIR="$CYPHER_DIR/Package_Relationship_Weights"
 PACKAGE_METRICS_CYPHER_DIR="$CYPHER_DIR/Metrics"
+EXTERNAL_DEPENDENCIES_CYPHER_DIR="$CYPHER_DIR/External_Dependencies"
 
 # Preparation - Create indizes
 execute_cypher "${CYPHER_DIR}/Create_index_for_full_qualified_type_name.cypher" || exit 1
@@ -44,3 +45,9 @@ execute_cypher_expect_results "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight10Percent
 # Preparation - Add Package node properties "incomingDependencies" and "outgoingDependencies"
 execute_cypher_expect_results "${PACKAGE_METRICS_CYPHER_DIR}/Set_Incoming_Package_Dependencies.cypher" || exit 1
 execute_cypher_expect_results "${PACKAGE_METRICS_CYPHER_DIR}/Set_Outgoing_Package_Dependencies.cypher" || exit 1
+
+# Preparation - Label external types and annotations
+#               "external" means that there is no byte code available, not a primitive type and not a java type
+#               "annoatation" means that there is a ANNOTATED_BY to that external type
+execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/Remove_external_type_and_annotation_labels.cypher" || exit 1
+execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/Label_external_types_and_annotations.cypher" || exit 1
diff --git a/scripts/reports/ExternalDependenciesCsv.sh b/scripts/reports/ExternalDependenciesCsv.sh
@@ -34,6 +34,11 @@ mkdir -p "${FULL_REPORT_DIRECTORY}"
 # Local Constants
 EXTERNAL_DEPENDENCIES_CYPHER_DIR="${CYPHER_DIR}/External_Dependencies"
 
+if ! execute_cypher_expect_results "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/List_external_types_used.cypher"; then
+    echo "Please execute 'prepareAnalysis.sh' with 'Label_external_types_and_annotations.cypher' first."
+    exit 1
+fi
+
 execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/External_package_usage_overall.cypher" > "${FULL_REPORT_DIRECTORY}/External_package_usage_overall.csv"
 execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/External_package_usage_per_type.cypher" > "${FULL_REPORT_DIRECTORY}/External_package_usage_per_type.csv"
 execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/External_package_usage_per_artifact.cypher" > "${FULL_REPORT_DIRECTORY}/External_package_usage_per_artifact.csv"

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+// List external types used`
	`2`	`+`
	`3`	`+MATCH (external:ExternalType) RETURN external.fqn`