Skip to content

Commit 1c0ff84

Browse files
JohTJohT
committed
Optimize Louvain Community Detection using Mutate
1 parent 8764313 commit 1c0ff84

9 files changed

+90
-12
lines changed

‎cypher/Community_Detection/Community_Detection_1a_Louvain_Estimate.cypher‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ CALL gds.louvain.write.estimate(
44
$dependencies_projection + '-without-empty', {
55
tolerance: 0.00001,
66
relationshipWeightProperty: $dependencies_projection_weight_property,
7-
writeProperty: 'louvainCommunityId',
7+
writeProperty: $dependencies_projection_write_property,
88
includeIntermediateCommunities: true
99
})
1010
YIELD nodeCount

‎cypher/Community_Detection/Community_Detection_1c_Louvain_Mutate.cypher‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
//Community Detection Louvain Mutate
2+
3+
CALL gds.louvain.mutate(
4+
$dependencies_projection + '-without-empty', {
5+
tolerance: 0.00001,
6+
consecutiveIds: NOT toBoolean($dependencies_include_intermediate_communities),
7+
includeIntermediateCommunities: toBoolean($dependencies_include_intermediate_communities),
8+
relationshipWeightProperty: $dependencies_projection_weight_property,
9+
mutateProperty: $dependencies_projection_write_property
10+
})
11+
YIELD communityCount
12+
,nodePropertiesWritten
13+
,ranLevels
14+
,modularity
15+
,modularities
16+
,preProcessingMillis
17+
,computeMillis
18+
,mutateMillis
19+
,postProcessingMillis
20+
,communityDistribution
21+
RETURN communityCount
22+
,nodePropertiesWritten
23+
,ranLevels
24+
,modularity
25+
,modularities
26+
,preProcessingMillis
27+
,computeMillis
28+
,mutateMillis
29+
,postProcessingMillis
30+
,communityDistribution.min
31+
,communityDistribution.mean
32+
,communityDistribution.max
33+
,communityDistribution.p50
34+
,communityDistribution.p75
35+
,communityDistribution.p90
36+
,communityDistribution.p95
37+
,communityDistribution.p99
38+
,communityDistribution.p999

‎cypher/Community_Detection/Community_Detection_1c_Louvain_Stream.cypher‎ renamed to ‎cypher/Community_Detection/Community_Detection_1d_Louvain_Stream.cypher‎

File renamed without changes.

‎cypher/Community_Detection/Community_Detection_1d_Louvain_Write_intermediateLouvainCommunityId.cypher‎ renamed to ‎cypher/Community_Detection/Community_Detection_1e_Louvain_Write_intermediateLouvainCommunityId.cypher‎

File renamed without changes.

‎cypher/Community_Detection/Community_Detection_1d_Louvain_Write_louvainCommunityId.cypher‎ renamed to ‎cypher/Community_Detection/Community_Detection_1e_Louvain_Write_louvainCommunityId.cypher‎

File renamed without changes.

‎cypher/Community_Detection/Community_Detection_1e_Louvain_Label_Delete.cypher‎ renamed to ‎cypher/Community_Detection/Community_Detection_1f_Louvain_Label_Delete.cypher‎

File renamed without changes.

‎cypher/Community_Detection/Community_Detection_1f_Louvain_Label.cypher‎ renamed to ‎cypher/Community_Detection/Community_Detection_1g_Louvain_Label.cypher‎

File renamed without changes.

‎cypher/Dependencies_Projection/Dependencies_8_Stream_Mutated.cypher‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ YIELD nodeId, nodeProperty, propertyValue
1010
,propertyValue
1111
OPTIONAL MATCH (artifact:Artifact)-[:CONTAINS]->(codeUnit)
1212
RETURN DISTINCT coalesce(codeUnit.fqn, codeUnit.fileName, codeUnit.name) AS codeUnitName
13-
,coalesce(replace(last(split(codeUnit.fileName, '/')), '.jar', ''), codeUnit.name) AS shortCodeUnitName
13+
,coalesce(codeUnit.name, replace(last(split(codeUnit.fileName, '/')), '.jar', '')) AS shortCodeUnitName
1414
,propertyName
1515
,propertyValue
16-
,coalesce(codeUnit.leidenCommunityId, 0) AS communityId
17-
,coalesce(codeUnit.centralityPageRank, 0.01) AS centrality
18-
,replace(last(split(artifact.fileName, '/')), '.jar', '') AS artifactName
16+
,coalesce(codeUnit.leidenCommunityId, 0) AS communityId // optional, might be null
17+
,coalesce(codeUnit.centralityPageRank, 0.01) AS centrality // optional, might be null
18+
,replace(last(split(artifact.fileName, '/')), '.jar', '') AS owningArtifactName

‎scripts/reports/CommunityCsv.sh‎

Lines changed: 47 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ detectCommunitiesWithLeiden() {
119119
# Label of the nodes that will be used for the projection. Example: "Package"
120120
# - dependencies_projection_weight_property=...
121121
# Name of the node property that contains the dependency weight. Example: "weight"
122-
detectCommunitiesWithLouvain() {
122+
detectCommunitiesWithLouvainOld() {
123123
local COMMUNITY_DETECTION_CYPHER_DIR="${CYPHER_DIR}/Community_Detection"
124124

125125
# Statistics
@@ -129,15 +129,55 @@ detectCommunitiesWithLouvain() {
129129
# Stream to CSV
130130
local nodeLabel
131131
nodeLabel=$( extractQueryParameter "dependencies_projection_node" "${@}")
132-
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1c_Louvain_Stream.cypher" "${@}" > "${FULL_REPORT_DIRECTORY}/${nodeLabel}_Communities_Louvain.csv"
132+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1d_Louvain_Stream.cypher" "${@}" > "${FULL_REPORT_DIRECTORY}/${nodeLabel}_Communities_Louvain.csv"
133133

134134
# Update Graph (node properties and labels)
135-
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1d_Louvain_Write_louvainCommunityId.cypher" "${@}"
136-
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1d_Louvain_Write_intermediateLouvainCommunityId.cypher" "${@}"
137-
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1e_Louvain_Label_Delete.cypher" "${@}"
138-
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1f_Louvain_Label.cypher" "${@}"
135+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1e_Louvain_Write_louvainCommunityId.cypher" "${@}"
136+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1e_Louvain_Write_intermediateLouvainCommunityId.cypher" "${@}"
137+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1f_Louvain_Label_Delete.cypher" "${@}"
138+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1g_Louvain_Label.cypher" "${@}"
139+
}
140+
141+
# Community Detection using the Louvain Algorithm
142+
#
143+
# Required Parameters:
144+
# - dependencies_projection=...
145+
# Name prefix for the in-memory projection name for dependencies. Example: "package"
146+
# - dependencies_projection_node=...
147+
# Label of the nodes that will be used for the projection. Example: "Package"
148+
# - dependencies_projection_weight_property=...
149+
# Name of the node property that contains the dependency weight. Example: "weight"
150+
detectCommunitiesWithLouvain() {
151+
local COMMUNITY_DETECTION_CYPHER_DIR="${CYPHER_DIR}/Community_Detection"
152+
local PROJECTION_CYPHER_DIR="${CYPHER_DIR}/Dependencies_Projection"
153+
local writePropertyName="dependencies_projection_write_property=louvainCommunityId"
154+
local writePropertyNameIntermediateCommunities="dependencies_projection_write_property=intermediateLouvainCommunityIds"
155+
local writeLabelName="dependencies_projection_write_label=LouvainCommunity"
156+
local includeIntermediateCommunities="dependencies_include_intermediate_communities=true"
157+
local excludeIntermediateCommunities="dependencies_include_intermediate_communities=false"
158+
159+
# Statistics
160+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1a_Louvain_Estimate.cypher" "${@}" "${writePropertyName}"
161+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1b_Louvain_Statistics.cypher" "${@}"
162+
163+
# Run the algorithm and write the result into the in-memory projection ("mutate")
164+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1c_Louvain_Mutate.cypher" "${@}" "${writePropertyName}" "${excludeIntermediateCommunities}"
165+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1c_Louvain_Mutate.cypher" "${@}" "${writePropertyNameIntermediateCommunities}" "${includeIntermediateCommunities}"
166+
167+
# Stream to CSV
168+
# TODO based on mutate both properties (communityId, intermediateCommunityId) possible?
169+
local nodeLabel
170+
nodeLabel=$( extractQueryParameter "dependencies_projection_node" "${@}")
171+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_1d_Louvain_Stream.cypher" "${@}" > "${FULL_REPORT_DIRECTORY}/${nodeLabel}_Communities_Louvain.csv"
172+
173+
# Update Graph (node properties and labels) using the already mutated property projection
174+
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_9_Write_Mutated.cypher" "${@}" "${writePropertyName}"
175+
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_9_Write_Mutated.cypher" "${@}" "${writePropertyNameIntermediateCommunities}"
176+
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_10_Delete_Label.cypher" "${@}" "${writePropertyName}" "${writeLabelName}"
177+
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_11_Add_Label.cypher" "${@}" "${writePropertyName}" "${writeLabelName}"
139178
}
140179

180+
141181
# Community Detection using the Weakly Connected Components Algorithm
142182
#
143183
# Required Parameters:
@@ -194,7 +234,7 @@ detectCommunitiesWithKCoreDecomposition() {
194234
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_8_Stream_Mutated.cypher" "${@}" "${writePropertyName}" > "${FULL_REPORT_DIRECTORY}/${nodeLabel}_Communities_K_Core_Decomposition.csv"
195235

196236
# Update Graph (node properties and labels) using the already mutated property projection
197-
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_9_Write_Mutated.cypher" "${@}" "${writePropertyName}" "${writeLabelName}"
237+
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_9_Write_Mutated.cypher" "${@}" "${writePropertyName}"
198238
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_10_Delete_Label.cypher" "${@}" "${writePropertyName}" "${writeLabelName}"
199239
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_11_Add_Label.cypher" "${@}" "${writePropertyName}" "${writeLabelName}"
200240
}

0 commit comments

Comments
 (0)