Skip to content

Commit 3db4134

Browse files
JohTJohT
committed
fixup! Optimize external dependencies report for > scale
1 parent 47de7d0 commit 3db4134

File tree

4 files changed

+103
-22
lines changed

4 files changed

+103
-22
lines changed

‎cypher/External_Dependencies/External_package_usage_per_artifact_distribution.cypher‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// External package usage per artifact distribution
2+
3+
MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
4+
MATCH (package)-[:CONTAINS]->(type:Type)
5+
WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
6+
,count(DISTINCT type.fqn) AS artifactTypes
7+
,count(DISTINCT package.fqn) AS artifactPackages
8+
,collect(type) AS typeList
9+
UNWIND typeList AS type
10+
MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
11+
MATCH (typePackage:Package)-[:CONTAINS]->(type)
12+
WHERE NOT externalType:ExternalAnnotation
13+
WITH artifactName
14+
,artifactTypes
15+
,artifactPackages
16+
,typePackage.fqn AS packageName
17+
,type.fqn AS fullTypeName
18+
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
19+
WITH artifactName
20+
,artifactPackages
21+
,artifactTypes
22+
,count(DISTINCT externalPackageName) AS numberOfExternalPackages
23+
,count(DISTINCT packageName) AS numberOfPackages
24+
,count(DISTINCT fullTypeName) AS numberOfTypes
25+
,COLLECT(DISTINCT packageName) AS nameOfPackages
26+
,COLLECT(DISTINCT fullTypeName) AS nameOfTypes
27+
RETURN artifactName
28+
,artifactTypes
29+
,artifactPackages
30+
,numberOfExternalPackages
31+
,numberOfPackages
32+
,numberOfTypes
33+
,100.0 / artifactTypes * numberOfTypes AS typesCallingExternalOnesRate
34+
,100.0 / artifactPackages * numberOfPackages AS packagesCallingExernalOnesRate
35+
,nameOfPackages
36+
,nameOfTypes[0..9] AS someTypeNames
37+
ORDER BY numberOfPackages DESC

‎cypher/External_Dependencies/External_package_usage_per_package.cypher‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// External package usage per package
2+
3+
MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
4+
MATCH (package)-[:CONTAINS]->(type:Type)
5+
WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
6+
,count(DISTINCT type.fqn) AS artifactTypes
7+
,count(DISTINCT package.fqn) AS artifactPackages
8+
,collect(type) AS typeList
9+
UNWIND typeList AS type
10+
MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
11+
MATCH (typePackage:Package)-[:CONTAINS]->(type)
12+
WHERE NOT externalType:ExternalAnnotation
13+
WITH artifactName
14+
,artifactPackages
15+
,artifactTypes
16+
,typePackage.fqn AS packageName
17+
,type.fqn AS fullTypeName
18+
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
19+
WITH artifactName
20+
,artifactPackages
21+
,artifactTypes
22+
,externalPackageName
23+
,count(DISTINCT packageName) AS numberOfPackages
24+
,COLLECT(DISTINCT packageName) AS nameOfPackages
25+
,count(DISTINCT fullTypeName) AS numberOfTypes
26+
,COLLECT(DISTINCT fullTypeName) AS nameOfTypes
27+
RETURN artifactName
28+
,artifactTypes
29+
,artifactPackages
30+
,externalPackageName
31+
,numberOfPackages
32+
,numberOfTypes
33+
,100.0 / artifactTypes * numberOfTypes AS typesCallingTheExternalPackageRate
34+
,100.0 / artifactPackages * numberOfPackages AS packagesCallingTheExernalPackageRate
35+
,nameOfPackages
36+
,nameOfTypes[0..9] AS someTypeNames
37+
ORDER BY numberOfPackages DESC, artifactName ASC

‎cypher/External_Dependencies/External_package_usage_per_type_distribution.cypher‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,38 @@
11
// External package usage per type distribution
22

3-
MATCH (artifact:Artifact)-[:CONTAINS]->(type:Type)
3+
MATCH (artifact:Artifact)-[:CONTAINS]->(package:Package)
4+
MATCH (package)-[:CONTAINS]->(type:Type)
45
WITH replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
5-
,count(type) AS artifactTypes
6+
,count(DISTINCT package.fqn) AS artifactPackages
7+
,count(DISTINCT type.fqn) AS artifactTypes
68
,collect(type) AS typeList
79
UNWIND typeList AS type
810
MATCH (type)-[:DEPENDS_ON]->(externalType:ExternalType)
9-
WHERE externalType:ExternalAnnotation
11+
MATCH (typePackage:Package)-[:CONTAINS]->(type)
12+
WHERE NOT externalType:ExternalAnnotation
1013
WITH artifactName
14+
,artifactPackages
1115
,artifactTypes
16+
,typePackage.fqn AS packageName
1217
,type.fqn AS fullTypeName
1318
,replace(externalType.fqn, '.' + externalType.name, '') AS externalPackageName
1419
WITH artifactName
20+
,artifactPackages
1521
,artifactTypes
16-
,fullTypeName
1722
,count(DISTINCT externalPackageName) AS numberOfExternalPackages
18-
WITH artifactName
19-
,artifactTypes
20-
,numberOfExternalPackages
23+
,COLLECT(DISTINCT externalPackageName) AS nameOfExternalPackages
24+
,count(DISTINCT packageName) AS numberOfPackages
25+
,COLLECT(DISTINCT packageName) AS nameOfPackages
2126
,count(DISTINCT fullTypeName) AS numberOfTypes
2227
,COLLECT(DISTINCT fullTypeName) AS nameOfTypes
2328
RETURN artifactName
29+
,artifactPackages
2430
,artifactTypes
2531
,numberOfExternalPackages
2632
,numberOfTypes
27-
,100.0 / artifactTypes * numberOfTypes AS numberOfTypesPercentage
28-
,nameOfTypes
29-
ORDER BY artifactName ASC, numberOfExternalPackages ASC
33+
,100.0 / artifactTypes * numberOfTypes AS typesCallingExternalRate
34+
,100.0 / artifactPackages * numberOfPackages AS packagesCallingExternalRate
35+
,nameOfExternalPackages[0..9] AS someExternalPackageNames
36+
,nameOfPackages[0..9] AS someExternalCallingPackageNames
37+
,nameOfTypes[0..9] AS someExternalCallingTypeNames
38+
ORDER BY numberOfExternalPackages DESC, artifactName ASC

‎jupyter/ExternalDependencies.ipynb‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,8 @@
425425
"outputs": [],
426426
"source": [
427427
"external_package_usage_per_type_distribution = query_cypher_to_data_frame(\"../cypher/External_Dependencies/External_package_usage_per_type_distribution.cypher\")\n",
428-
"external_package_usage_per_type_distribution[['artifactName', 'artifactTypes', 'numberOfExternalPackages', 'numberOfTypes', 'numberOfTypesPercentage']].head(20)"
428+
"external_package_usage_per_type_distribution_truncated=external_package_usage_per_type_distribution[['artifactName', 'artifactPackages', 'artifactTypes', 'numberOfExternalPackages', 'numberOfTypes', 'typesCallingExternalRate', 'packagesCallingExternalRate']].head(20)\n",
429+
"external_package_usage_per_type_distribution_truncated"
429430
]
430431
},
431432
{
@@ -447,17 +448,14 @@
447448
"outputs": [],
448449
"source": [
449450
"\n",
450-
"# Organize artifacts in columns with the number of types as values using pivot\n",
451+
"# Organize artifacts in columns with the number of package that call external packages as values using pivot\n",
451452
"# Every row represents the number of external packages\n",
452-
"external_package_usage_per_type_distribution=external_package_usage_per_type_distribution.pivot(index='numberOfExternalPackages', columns='artifactName', values='numberOfTypesPercentage')\n",
453+
"external_package_usage_per_type_distribution_for_each_artifact=external_package_usage_per_type_distribution.pivot(index='numberOfExternalPackages', columns='artifactName', values='packagesCallingExternalRate')\n",
453454
"\n",
454455
"# Fill missing values with zero\n",
455-
"external_package_usage_per_type_distribution.fillna(0, inplace=True)\n",
456+
"external_package_usage_per_type_distribution_for_each_artifact.fillna(0, inplace=True)\n",
456457
"\n",
457-
"# Convert to integer\n",
458-
"# external_package_usage_per_type_distribution=external_package_usage_per_type_distribution.astype(int)\n",
459-
"\n",
460-
"external_package_usage_per_type_distribution.head(10)"
458+
"external_package_usage_per_type_distribution_for_each_artifact"
461459
]
462460
},
463461
{
@@ -479,12 +477,12 @@
479477
"outputs": [],
480478
"source": [
481479
"plot.figure();\n",
482-
"axes = external_package_usage_per_type_distribution.plot(\n",
480+
"axes = external_package_usage_per_type_distribution_for_each_artifact.plot(\n",
483481
" kind='bar', \n",
484482
" grid=True,\n",
485483
" title='Relative External Package Usage', \n",
486484
" xlabel='external package count',\n",
487-
" ylabel='percentage of types',\n",
485+
" ylabel='percentage of package',\n",
488486
" cmap=main_color_map,\n",
489487
")\n",
490488
"plot.show()"
@@ -509,12 +507,12 @@
509507
"outputs": [],
510508
"source": [
511509
"plot.figure();\n",
512-
"axes = external_package_usage_per_type_distribution.transpose().plot(\n",
510+
"axes = external_package_usage_per_type_distribution_for_each_artifact.transpose().plot(\n",
513511
" kind='bar', \n",
514512
" grid=True,\n",
515513
" title='Relative External Package Usage', \n",
516514
" xlabel='artifact',\n",
517-
" ylabel='percentage of types',\n",
515+
" ylabel='percentage of packages',\n",
518516
" stacked=True,\n",
519517
" cmap=main_color_map,\n",
520518
")\n",

0 commit comments

Comments
 (0)