Skip to content

Commit 77f5043

Browse files
JohTJohT
committed
Optimize external dependencies report for > scale
1 parent 36ed40b commit 77f5043

16 files changed

+1534
-135
lines changed

‎cypher/External_Dependencies/External_package_levels.cypher‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// External package levels
2+
3+
MATCH (externalType:ExternalType)
4+
WITH replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
5+
WITH count(DISTINCT split(externalPackageName, '.')[0]) AS externalFirstLevelPackages
6+
,count(DISTINCT split(externalPackageName, '.')[0..1]) AS externalSecondLevelPackages
7+
,count(DISTINCT split(externalPackageName, '.')[0..2]) AS externalThirdLevelPackages
8+
,count(DISTINCT split(externalPackageName, '.')[0..3]) AS externalForthLevelPackages
9+
,count(DISTINCT split(externalPackageName, '.')[0..4]) AS externalFifthLevelPackages
10+
,count(DISTINCT externalPackageName) AS allExternalPackages
11+
RETURN externalFirstLevelPackages
12+
,externalSecondLevelPackages
13+
,externalThirdLevelPackages
14+
,externalForthLevelPackages
15+
,externalFifthLevelPackages
16+
,allExternalPackages

‎cypher/External_Dependencies/External_package_name_elements.cypher‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// External package name elements
2+
3+
MATCH (externalType:ExternalType)
4+
WITH replace(externalType.fqn, '.' + externalType.name, '') AS packageName
5+
WITH size(split(packageName,'.')) AS packageNameElements
6+
,count(DISTINCT packageName) AS packageCount
7+
,collect(DISTINCT packageName)[0..19] AS somePackageNames
8+
RETURN packageNameElements
9+
,packageCount
10+
,somePackageNames
11+
ORDER BY packageNameElements

‎cypher/External_Dependencies/External_package_usage_overall.cypher‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,27 @@
11
// External package usage overall
22

3-
MATCH (type:Type)
4-
WITH count(type) as allTypes, collect(type) as typeList
3+
MATCH (package:Package)-[:CONTAINS]->(type:Type)
4+
WITH count(DISTINCT type.fqn) AS allTypes
5+
,count(DISTINCT package.fqn) AS allPackages
6+
,collect(type) as typeList
57
UNWIND typeList AS type
68
MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
9+
MATCH (typePackage:Package)-[:CONTAINS]->(type)
710
WITH allTypes
11+
,allPackages
812
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
9-
,count(externalDependency) AS numberOfExternalTypeCaller
10-
,sum(externalDependency.weight) AS numberOfExternalTypeCalls
13+
,count(DISTINCT typePackage.fqn) AS numberOfExternalCallerPackages
14+
,count(DISTINCT type.fqn) AS numberOfExternalCallerTypes
15+
,count(externalDependency) AS numberOfExternalTypeCalls
16+
,sum(externalDependency.weight) AS numberOfExternalTypeCallsWeighted
1117
,collect(DISTINCT externalType.name) AS externalTypeNames
18+
where numberOfExternalTypeCalls <> numberOfExternalCallerTypes
1219
RETURN externalPackageName
13-
,numberOfExternalTypeCaller
20+
,numberOfExternalCallerPackages
21+
,numberOfExternalCallerTypes
1422
,numberOfExternalTypeCalls
23+
,numberOfExternalTypeCallsWeighted
24+
,allPackages
1525
,allTypes
16-
,externalTypeNames
17-
ORDER BY numberOfExternalTypeCaller DESC, externalPackageName ASC
26+
,externalTypeNames[0..9] AS tenExternalTypeNames
27+
ORDER BY numberOfExternalCallerPackages DESC, externalPackageName ASC

‎cypher/External_Dependencies/External_package_usage_per_artifact_and_external_package.cypher‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// External package usage per artifact and external package
2+
3+
// Get the overall artifact statistics first
4+
MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
5+
MATCH (package)-[:CONTAINS]->(type:Type)
6+
OPTIONAL MATCH (packageUsingExternal:Package)-[:CONTAINS]->(type)-[:DEPENDS_ON]->(external:ExternalType)
7+
WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
8+
,count(DISTINCT package.fqn) AS artifactPackages
9+
,count(DISTINCT type.fqn) AS artifactTypes
10+
,count(DISTINCT replace(external.fqn, '.' + external.name, '')) AS artifactExternalPackages
11+
,count(DISTINCT packageUsingExternal.fqn) AS artifactExternalCallingPackages
12+
,collect(type) AS typeList
13+
WITH artifactName
14+
,artifactPackages
15+
,artifactTypes
16+
,artifactExternalPackages
17+
,artifactExternalCallingPackages
18+
,round((100.0 / artifactPackages * artifactExternalCallingPackages), 2) AS artifactExternalCallingPackagesRate
19+
,typeList
20+
// Get the external dependencies for each internal type
21+
UNWIND typeList AS type
22+
MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
23+
MATCH (typePackage:Package)-[:CONTAINS]->(type)
24+
// Optionally filter out dependencies to external annotations
25+
// WHERE NOT externalType:ExternalAnnotation
26+
WITH artifactName
27+
,artifactPackages
28+
,artifactTypes
29+
,artifactExternalPackages
30+
,artifactExternalCallingPackages
31+
,artifactExternalCallingPackagesRate
32+
,typePackage.fqn AS packageName
33+
,type.fqn AS fullTypeName
34+
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
35+
// Group by artifact and external package
36+
RETURN artifactName
37+
,artifactPackages
38+
,artifactTypes
39+
,artifactExternalPackages
40+
,artifactExternalCallingPackages
41+
,artifactExternalCallingPackagesRate
42+
,externalPackageName
43+
,count(DISTINCT packageName) AS numberOfPackages
44+
,count(DISTINCT fullTypeName) AS numberOfTypes
45+
,100.0 / artifactPackages * count(DISTINCT packageName) AS packagesCallingExternalRate
46+
,100.0 / artifactTypes * count(DISTINCT fullTypeName) AS typesCallingExternalRate
47+
,COLLECT(DISTINCT packageName) AS nameOfPackages
48+
,COLLECT(DISTINCT fullTypeName)[0..9] AS someTypeNames
49+
// Order the results by number of packages that use the external package dependency descending
50+
ORDER BY artifactExternalCallingPackagesRate DESC, artifactName ASC, numberOfPackages DESC, externalPackageName ASC

‎cypher/External_Dependencies/External_package_usage_per_artifact_distribution.cypher‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// External package usage per artifact distribution
2+
3+
MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
4+
MATCH (package)-[:CONTAINS]->(type:Type)
5+
WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
6+
,count(DISTINCT package.fqn) AS artifactPackages
7+
,count(DISTINCT type.fqn) AS artifactTypes
8+
,collect(type) AS typeList
9+
UNWIND typeList AS type
10+
MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
11+
MATCH (typePackage:Package)-[:CONTAINS]->(type)
12+
WHERE NOT externalType:ExternalAnnotation
13+
WITH artifactName
14+
,artifactPackages
15+
,artifactTypes
16+
,typePackage.fqn AS packageName
17+
,type.fqn AS fullTypeName
18+
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
19+
WITH artifactName
20+
,artifactPackages
21+
,artifactTypes
22+
,count(DISTINCT externalPackageName) AS numberOfExternalPackages
23+
,COLLECT(DISTINCT externalPackageName) AS nameOfExternalPackages
24+
,count(DISTINCT packageName) AS numberOfPackages
25+
,COLLECT(DISTINCT packageName) AS nameOfPackages
26+
,count(DISTINCT fullTypeName) AS numberOfTypes
27+
,COLLECT(DISTINCT fullTypeName) AS nameOfTypes
28+
RETURN artifactName
29+
,artifactPackages
30+
,artifactTypes
31+
,numberOfExternalPackages
32+
,numberOfPackages
33+
,numberOfTypes
34+
,100.0 / artifactTypes * numberOfTypes AS typesCallingExternalRate
35+
,100.0 / artifactPackages * numberOfPackages AS packagesCallingExternalRate
36+
,nameOfExternalPackages[0..9] AS someExternalPackageNames
37+
,nameOfPackages[0..9] AS someExternalCallingPackageNames
38+
,nameOfTypes[0..9] AS someExternalCallingTypeNames
39+
ORDER BY numberOfPackages DESC, artifactName ASC

‎cypher/External_Dependencies/External_package_usage_per_artifact_package_aggregated.cypher‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// External package usage per artifact package aggregated
2+
3+
// Get the overall artifact statistics first
4+
MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
5+
MATCH (package)-[:CONTAINS]->(type:Type)
6+
WHERE NOT type:ExternalType
7+
WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
8+
,artifact.leidenCommunityId AS leidenCommunityId
9+
,count(DISTINCT package.fqn) AS artifactPackages
10+
,count(DISTINCT type.fqn) AS artifactTypes
11+
,collect(type) AS typeList
12+
// Get the external dependencies for each internal type
13+
UNWIND typeList AS type
14+
MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
15+
MATCH (typePackage:Package)-[:CONTAINS]->(type)
16+
// Filter out dependencies to exxternal annotations
17+
WHERE NOT externalType:ExternalAnnotation
18+
WITH artifactName
19+
,leidenCommunityId
20+
,artifactPackages
21+
,artifactTypes
22+
,typePackage.fqn AS packageName
23+
,type.fqn AS fullTypeName
24+
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
25+
// Group by artifact and external package
26+
WITH artifactName
27+
,leidenCommunityId
28+
,artifactPackages
29+
,artifactTypes
30+
,externalPackageName
31+
,count(DISTINCT packageName) AS numberOfPackages
32+
,COLLECT(DISTINCT packageName) AS nameOfPackages
33+
,count(DISTINCT fullTypeName) AS numberOfTypes
34+
,COLLECT(DISTINCT fullTypeName) AS nameOfTypes
35+
,100.0 / artifactPackages * count(DISTINCT packageName) AS packagesCallingExternalRate
36+
,100.0 / artifactTypes * count(DISTINCT fullTypeName) AS typesCallingExternalRate
37+
// Pre order the results by number of packages that use the external package dependency descending
38+
ORDER BY numberOfPackages DESC, artifactName ASC
39+
// Optionally filter out external package dependencies that are only used by one package
40+
// WHERE numberOfPackages > 1
41+
// Group by artifact, aggregate statistics and return the results
42+
RETURN artifactName
43+
,leidenCommunityId
44+
,artifactPackages
45+
,artifactTypes
46+
,count(DISTINCT externalPackageName) AS numberOfExternalPackages
47+
48+
// Statistics about the packages and their external package usage count
49+
,min(numberOfPackages) AS minNumberOfPackages
50+
,max(numberOfPackages) AS maxNumberOfPackages
51+
,percentileCont(numberOfPackages, 0.5) AS medNumberOfPackages
52+
,avg(numberOfPackages) AS avgNumberOfPackages
53+
,stDev(numberOfPackages) AS stdNumberOfPackages
54+
55+
// Statistics about the packages and their external package usage percentage
56+
,min(packagesCallingExternalRate) AS minNumberOfPackagesPercentage
57+
,max(packagesCallingExternalRate) AS maxNumberOfPackagesPercentage
58+
,percentileCont(packagesCallingExternalRate, 0.5) AS medNumberOfPackagesPercentage
59+
,avg(packagesCallingExternalRate) AS avgNumberOfPackagesPercentage
60+
,stDev(packagesCallingExternalRate) AS stdNumberOfPackagesPercentage
61+
62+
// Statistics about the types and their external package usage count
63+
,min(numberOfTypes) AS minNumberOfTypes
64+
,max(numberOfTypes) AS maxNumberOfTypes
65+
,percentileCont(numberOfTypes, 0.5) AS medNumberOfTypes
66+
,avg(numberOfTypes) AS avgNumberOfTypes
67+
,stDev(numberOfTypes) AS stdNumberOfTypes
68+
69+
// Statistics about the types and their external package usage count percentage
70+
,min(typesCallingExternalRate) AS minNumberOfTypesPercentage
71+
,max(typesCallingExternalRate) AS maxNumberOfTypesPercentage
72+
,percentileCont(typesCallingExternalRate, 0.5) AS medNumberOfTypesPercentage
73+
,avg(typesCallingExternalRate) AS avgNumberOfTypesPercentage
74+
,stDev(typesCallingExternalRate) AS stdNumberOfTypesPercentage
75+
76+
// Examples of external packages, caller packages and caller types
77+
,collect(externalPackageName)[0..9] AS top10ExternalPackageNamesByUsageDescending
78+
,COLLECT(nameOfPackages)[0][0..9] AS somePackageNames
79+
,COLLECT(nameOfTypes)[0][0..9] AS someTypeNames
80+
81+
ORDER BY maxNumberOfPackages DESC, artifactName ASC

‎cypher/External_Dependencies/External_package_usage_per_artifact_sorted.cypher‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// External package usage per artifact sorted by external usage descending
2+
3+
MATCH (artifact:Artifact:Archive)-[:CONTAINS]->(type:Type)
4+
OPTIONAL MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
5+
WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
6+
,count(DISTINCT type.fqn) AS numberOfTypesInArtifact
7+
,count(DISTINCT externalType.fqn) AS numberOfExternalTypesInArtifact
8+
,count(DISTINCT replace(externalType.fqn, '.' + externalType.name, '')) AS numberOfExternalPackagesInArtifact
9+
,collect(DISTINCT type) AS typeList
10+
UNWIND typeList AS type
11+
MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
12+
WITH numberOfTypesInArtifact
13+
,numberOfExternalTypesInArtifact
14+
,numberOfExternalPackagesInArtifact
15+
,100.0 / numberOfTypesInArtifact * numberOfExternalTypesInArtifact AS externalTypeRate
16+
,externalDependency
17+
,artifactName
18+
,type.fqn AS fullTypeName
19+
,type.name AS typeName
20+
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
21+
,externalType.name AS externalTypeName
22+
WITH numberOfTypesInArtifact
23+
,numberOfExternalTypesInArtifact
24+
,numberOfExternalPackagesInArtifact
25+
,externalTypeRate
26+
,artifactName
27+
,externalPackageName
28+
,count(externalDependency) AS numberOfExternalTypeCaller
29+
,sum(externalDependency.weight) AS numberOfExternalTypeCalls
30+
,collect(DISTINCT externalTypeName) AS externalTypeNames
31+
RETURN artifactName
32+
,externalPackageName
33+
,numberOfExternalTypeCaller
34+
,numberOfExternalTypeCalls
35+
,numberOfTypesInArtifact
36+
,numberOfExternalTypesInArtifact
37+
,numberOfExternalPackagesInArtifact
38+
,externalTypeRate
39+
,externalTypeNames
40+
ORDER BY externalTypeRate DESC, artifactName ASC, numberOfExternalTypeCaller DESC, externalPackageName ASC

‎cypher/External_Dependencies/External_package_usage_per_artifact_sorted_top.cypher‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// External package usage per artifact top externals
2+
3+
MATCH (artifact:Artifact:Archive)-[:CONTAINS]->(type:Type)
4+
OPTIONAL MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
5+
WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
6+
,count(DISTINCT type.fqn) AS numberOfTypesInArtifact
7+
,count(DISTINCT externalType.fqn) AS numberOfExternalTypesInArtifact
8+
,count(DISTINCT replace(externalType.fqn, '.' + externalType.name, '')) AS numberOfExternalPackagesInArtifact
9+
,collect(DISTINCT type) AS typeList
10+
UNWIND typeList AS type
11+
MATCH (type)-[externalDependency:DEPENDS_ON]->(externalType:ExternalType)
12+
WITH numberOfTypesInArtifact
13+
,numberOfExternalTypesInArtifact
14+
,numberOfExternalPackagesInArtifact
15+
,100.0 / numberOfTypesInArtifact * numberOfExternalTypesInArtifact AS externalTypeRate
16+
,externalDependency
17+
,artifactName
18+
,type.fqn AS fullTypeName
19+
,type.name AS typeName
20+
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
21+
,externalType.name AS externalTypeName
22+
ORDER BY externalTypeRate DESC, artifactName ASC
23+
WITH numberOfTypesInArtifact
24+
,numberOfExternalTypesInArtifact
25+
,numberOfExternalPackagesInArtifact
26+
,externalTypeRate
27+
,artifactName
28+
,externalPackageName
29+
,count(externalDependency) AS numberOfExternalTypeCaller
30+
,sum(externalDependency.weight) AS numberOfExternalTypeCalls
31+
,collect(DISTINCT externalTypeName) AS externalTypeNames
32+
ORDER BY externalTypeRate DESC, artifactName ASC, numberOfExternalTypeCaller DESC
33+
WITH numberOfTypesInArtifact
34+
,numberOfExternalTypesInArtifact
35+
,numberOfExternalPackagesInArtifact
36+
,externalTypeRate
37+
,artifactName
38+
,COLLECT(DISTINCT externalPackageName) AS externalPackageNames
39+
,SUM(numberOfExternalTypeCaller) AS numberOfExternalTypeCaller
40+
,sum(numberOfExternalTypeCalls) AS numberOfExternalTypeCalls
41+
,collect(externalTypeNames) AS externalTypeNames
42+
RETURN artifactName
43+
,numberOfTypesInArtifact
44+
,numberOfExternalTypesInArtifact
45+
,numberOfExternalPackagesInArtifact
46+
,externalTypeRate
47+
,numberOfExternalTypeCaller
48+
,numberOfExternalTypeCalls
49+
,size(externalPackageNames) AS numberOfExternalPackages
50+
,externalPackageNames[0..4] AS top5ExternalPackages
51+
,externalTypeNames[0..1] AS someExternalTypes
52+
LIMIT 40

‎cypher/External_Dependencies/External_package_usage_per_internal_package_count.cypher‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// External package usage per internal package count
2+
3+
// Get the overall artifact statistics first
4+
MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
5+
MATCH (package)-[:CONTAINS]->(type:Type)
6+
WHERE NOT type:ExternalType
7+
WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
8+
,count(DISTINCT package.fqn) AS artifactPackages
9+
,count(DISTINCT type.fqn) AS artifactTypes
10+
,collect(type) AS typeList
11+
// Get the external dependencies for each internal type
12+
UNWIND typeList AS type
13+
MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
14+
MATCH (typePackage:Package)-[:CONTAINS]->(type)
15+
// Optionally filter out dependencies to external annotations
16+
// WHERE NOT externalType:ExternalAnnotation
17+
WITH artifactName
18+
,artifactPackages
19+
,artifactTypes
20+
,typePackage.fqn AS packageName
21+
,type.fqn AS fullTypeName
22+
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
23+
// Group by artifact and external package
24+
WITH artifactName
25+
,artifactPackages
26+
,artifactTypes
27+
,externalPackageName
28+
,count(DISTINCT packageName) AS numberOfPackages
29+
,count(DISTINCT fullTypeName) AS numberOfTypes
30+
,100.0 / artifactPackages * count(DISTINCT packageName) AS packagesCallingExternalRate
31+
,100.0 / artifactTypes * count(DISTINCT fullTypeName) AS typesCallingExternalRate
32+
,COLLECT(DISTINCT packageName) AS nameOfPackages
33+
,COLLECT(DISTINCT fullTypeName)[0..9] AS someTypeNames
34+
RETURN artifactName
35+
,artifactPackages
36+
,artifactTypes
37+
,numberOfPackages
38+
,count(DISTINCT externalPackageName) AS numberOfExternalPackages
39+
,collect(DISTINCT externalPackageName) AS externalPackageNames
40+
,max(packagesCallingExternalRate) AS maxPackagesCallingExternalRate
41+
,max(typesCallingExternalRate) AS maxTypesCallingExternalRate
42+
,COLLECT(nameOfPackages)[0][0..9] AS somePackageNames
43+
,COLLECT(someTypeNames)[0] AS someTypeNames
44+
// Order the results by number of packages that use the external package dependency descending
45+
ORDER BY numberOfPackages DESC, artifactName ASC

0 commit comments

Comments
 (0)