Graph Visualization with GraphViz

.github/workflows/java-code-analysis.yml

@@ -69,15 +69,6 @@ jobs:
         distribution: 'adopt'
         java-version: ${{ matrix.java }}
 
-    - name: Setup Node.js for Graph Visualization
-      uses: actions/setup-node@v4
-      with:
-        node-version-file: 'graph-visualization/.nvmrc'
-
-    - name: Install Node packages for Graph Visualization
-      working-directory: graph-visualization
-      run: npm ci
-
     - name: Setup Cache for Conda package manager Miniforge
       uses: actions/cache@v4
       env:
@@ -142,7 +133,7 @@ jobs:
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
+        name: java-code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
         path: |
           ./temp/**/runtime/*
           ./temp/**/reports/*
@@ -153,8 +144,8 @@ jobs:
       if: success()
       uses: actions/upload-artifact@v4
       with:
-        name: code-report-results-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
-        path: ./results
+        name: java-code-analysis-results-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
+        path: ./temp/**/reports/*
         if-no-files-found: error
         retention-days: 5
 
@@ -164,7 +155,7 @@ jobs:
     #- name: Archive exported database
     #  uses: actions/upload-artifact@v3
     #  with:
-    #    name: code-report-database-export-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
+    #    name: java-code-analysis-database-export-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
     #    path: ./temp/**/import
     #    if-no-files-found: error
     #    retention-days: 5

.github/workflows/typescript-code-analysis.yml

@@ -69,15 +69,6 @@ jobs:
         distribution: 'adopt'
         java-version: ${{ matrix.java }}
 
-    - name: Setup Node.js for Graph Visualization
-      uses: actions/setup-node@v4
-      with:
-        node-version-file: 'graph-visualization/.nvmrc'
-
-    - name: Install Node packages for Graph Visualization
-      working-directory: graph-visualization
-      run: npm ci
-
     - name: Setup Cache for Conda package manager Miniforge
       uses: actions/cache@v4
       env:
@@ -156,7 +147,7 @@ jobs:
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
+        name: typescript-code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
         path: |
           ./temp/**/runtime/*
           ./temp/**/reports/*
@@ -167,8 +158,8 @@ jobs:
       if: success()
       uses: actions/upload-artifact@v4
       with:
-        name: code-report-results-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
-        path: ./results
+        name: typescript-code-analysis-results-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
+        path: ./temp/**/reports/*
         if-no-files-found: error
         retention-days: 5
 
@@ -178,7 +169,7 @@ jobs:
     #- name: Archive exported database
     #  uses: actions/upload-artifact@v3
     #  with:
-    #    name: code-report-database-export-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
+    #    name: typescript-code-analysis-database-export-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}
     #    path: ./temp/**/import
     #    if-no-files-found: error
     #    retention-days: 5

README.md

@@ -124,6 +124,8 @@ The [Code Structure Analysis Pipeline](./.github/workflows/java-code-analysis.ym
   - [openTSNE](https://github.com/pavlin-policar/openTSNE)
   - [wordcloud](https://github.com/amueller/word_cloud)
 - [Graph Visualization](./graph-visualization/README.md) uses [node.js](https://nodejs.org/de) and the dependencies listed in [package.json](./graph-visualization/package.json).
+- [HPCC-Systems (High Performance Computing Cluster) Web-Assembly (JavaScript)](https://github.com/hpcc-systems/hpcc-js-wasm) containing a wrapper for GraphViz to visualize graph structures.
+- [GraphViz](https://gitlab.com/graphviz/graphviz) for CLI Graph Visualization
 - [Check links in markdown documentation (GitHub workflow)](./.github/workflows/check-links-in-documentation.yml) uses [markdown-link-check](https://github.com/tcort/markdown-link-check).
 
 **Big shout-out** 📣 to all the creators and contributors of these great libraries 👍. Projects like this wouldn't be possible without them. Feel free to [create an issue](https://github.com/JohT/code-graph-analysis-pipeline/issues/new/choose) if something is missing or wrong in the list.

cypher/Internal_Dependencies/Java_Artifact_build_levels_for_graphviz.cypher

@@ -0,0 +1,31 @@
+// List of all Java Artifacts and their dependencies with build levels for GraphViz Visualization
+
+ MATCH (sourceForStatistics:Java:Artifact)-[dependencyForStatistics:DEPENDS_ON]->(targetForStatistics:Java:Artifact)
+ WHERE sourceForStatistics.maxDistanceFromSource IS NOT NULL
+   AND targetForStatistics.maxDistanceFromSource IS NOT NULL
+  WITH min(dependencyForStatistics.weight) AS minWeight
+      ,max(dependencyForStatistics.weight) AS maxWeight
+      ,max(targetForStatistics.maxDistanceFromSource) AS maxLevel
+ MATCH (source:Java:Artifact)-[dependency:DEPENDS_ON]->(target:Java:Artifact)
+ WHERE source.maxDistanceFromSource IS NOT NULL
+   AND target.maxDistanceFromSource IS NOT NULL
+  WITH *, toFloat(dependency.weight - minWeight) / toFloat(maxWeight - minWeight) AS normalizedWeight
+  WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth
+  WITH *, source.name + "\\n(level " + coalesce(source.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS fullSourceName
+  WITH *, target.name + "\\n(level " + coalesce(target.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS fullTargetName
+  WITH *, "\" -> \"" + fullTargetName 
+                   + "\" [label = "  + dependency.weight + ";"
+                   + " penwidth = "  + penWidth + ";"
+                   + " ];"    AS graphVizDotNotationEdge
+  WITH *, "\"" + fullSourceName + coalesce(graphVizDotNotationEdge, "\" [];") AS graphVizDotNotationLine
+ ORDER BY dependency.weight DESC, target.maxDistanceFromSource DESC
+RETURN graphVizDotNotationLine
+      //Debugging
+      //,source.name       AS sourceName
+      //,target.name       AS targetName
+      //,penWidth
+      //,normalizedWeight
+      //,dependency.weight AS weight
+      //,minWeight
+      //,maxWeight
+LIMIT 440

cypher/Internal_Dependencies/Typescript_Module_build_levels_for_graphviz.cypher

@@ -0,0 +1,31 @@
+// List of all Typescript modules and their dependencies with build levels for GraphViz Visualization
+
+ MATCH (sourceForStatistics:TS:Module)-[dependencyForStatistics:DEPENDS_ON]->(targetForStatistics:TS:Module)
+ WHERE sourceForStatistics.maxDistanceFromSource IS NOT NULL
+   AND targetForStatistics.maxDistanceFromSource IS NOT NULL
+  WITH min(dependencyForStatistics.weight) AS minWeight
+      ,max(dependencyForStatistics.weight) AS maxWeight
+      ,max(targetForStatistics.maxDistanceFromSource) AS maxLevel
+MATCH (source:TS:Module)-[dependency:DEPENDS_ON]->(target:TS:Module)
+ WHERE source.maxDistanceFromSource IS NOT NULL
+   AND target.maxDistanceFromSource IS NOT NULL
+  WITH *, toFloat(dependency.cardinality - minWeight) / toFloat(maxWeight - minWeight) AS normalizedWeight
+  WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth
+  WITH *, source.rootProjectName + "\\n" + source.name + "\\n(level " + coalesce(source.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS fullSourceName
+  WITH *, target.rootProjectName + "\\n" + target.name + "\\n(level " + coalesce(target.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS fullTargetName
+  WITH *, "\" -> \"" + fullTargetName 
+                   + "\" [label = "  + dependency.cardinality + ";"
+                   + " penwidth = "  + penWidth + ";"
+                   + " ];"    AS graphVizDotNotationEdge
+  WITH *, "\"" + fullSourceName + coalesce(graphVizDotNotationEdge, "\" [];") AS graphVizDotNotationLine
+ ORDER BY dependency.weight DESC, target.maxDistanceFromSource DESC
+RETURN graphVizDotNotationLine
+      //Debugging
+      //,source.name       AS sourceName
+      //,target.name       AS targetName
+      //,penWidth
+      //,normalizedWeight
+      //,dependency.cardinality AS weight
+      //,minWeight
+      //,maxWeight
+LIMIT 440

cypher/Path_Finding/Path_Finding_6_Longest_paths_contributors_for_graphviz.cypher

@@ -0,0 +1,77 @@
+// Path Finding - Longest path - Stream - List all dependencies for nodes contributing to longest paths and highlight those paths in the Visualization with GraphViz.
+
+// Gather global statistics about dependency weights and levels for normalization and node details
+ MATCH (sourceNodeForStatistics)-[dependencyForStatistics:DEPENDS_ON]->(targetNodeForStatistics)
+ WHERE $dependencies_projection_node IN LABELS(sourceNodeForStatistics)
+   AND $dependencies_projection_node IN LABELS(targetNodeForStatistics)
+  WITH min(dependencyForStatistics[$dependencies_projection_weight_property]) AS minWeight
+      ,max(dependencyForStatistics[$dependencies_projection_weight_property]) AS maxWeight
+      ,max(targetNodeForStatistics.maxDistanceFromSource)                     AS maxLevel
+   WITH *, 1.0 / toFloat(maxWeight - minWeight)                               AS weightNormalizationFactor
+   WITH { minWeight: minWeight, maxLevel: maxLevel, weightNormalizationFactor: weightNormalizationFactor } AS statistics
+// -> Main call to execute "longest path" algorithm
+   CALL gds.dag.longestPath.stream($dependencies_projection + '-cleaned')
+  YIELD index, totalCost, path
+   WITH *
+// Sort longest paths by their length descending and - if equal - by their index ascending
+  ORDER BY totalCost DESC, index ASC
+// Only take the top 50 longest paths as a compromise between performance and visualization content
+  LIMIT 50
+// Collect all results of the longest path search as well as all nodes of the longest paths
+   WITH statistics
+       ,collect({index: index, distance: toInteger(totalCost), path: path}) AS longestPaths
+       ,collect(nodes(path)) AS allLongestPathNodes
+// Flatten and deduplicate the list of all nodes that contribute to at least one longest path
+ UNWIND allLongestPathNodes AS longestPathNodes
+ UNWIND longestPathNodes    AS longestPathNode
+   WITH statistics
+       ,longestPaths
+       ,collect(DISTINCT longestPathNode) AS allDistinctLongestPathNodes
+// Iterate over all longest paths
+ UNWIND longestPaths AS longestPath
+   WITH statistics
+       ,longestPaths, allDistinctLongestPathNodes
+       ,[ singleRelationship IN relationships(longestPath.path)     | [startNode(singleRelationship), endNode(singleRelationship)] ] AS allLongestPathStartAndEndNodeTuples
+       ,[ singleRelationship IN relationships(longestPaths[0].path) | [startNode(singleRelationship), endNode(singleRelationship)] ] AS longestPathStartAndEndNodeTuples
+       ,longestPath.index       AS index
+       ,longestPath.distance    AS distance
+// -> Main query of all dependencies of nodes contributing to the longest paths
+ MATCH (source)-[dependency:DEPENDS_ON]->(target)
+ WHERE $dependencies_projection_node IN labels(source)
+   AND $dependencies_projection_node IN labels(target)
+   // Dependent nodes need to be part of at least one longest paths
+   AND (source IN allDistinctLongestPathNodes AND target IN allDistinctLongestPathNodes)
+  WITH statistics.maxLevel                  AS maxLevel
+      ,statistics.minWeight                 AS minWeight
+      ,statistics.weightNormalizationFactor AS weightNormalizationFactor
+      ,count(index)                         AS numberOfLongestPathsPassing
+      ,max(distance)                        AS lengthOfLongestPathPassing
+      ,dependency
+      ,source
+      ,target
+      // If there is at least one longest path passing through the dependency then "contributesToALongestPath" is true
+      ,([source, target] IN allLongestPathStartAndEndNodeTuples)  AS contributesToALongestPath
+      ,([source, target] IN longestPathStartAndEndNodeTuples)     AS isPartOfLongestPath
+  WITH *, dependency[$dependencies_projection_weight_property]    AS weight
+  WITH *, toFloat(weight - minWeight) * weightNormalizationFactor AS normalizedWeight
+  WITH *, round((normalizedWeight * 5) + 1, 2)                    AS penWidth
+  WITH *, source.name + "\\n(level " + source.maxDistanceFromSource + "/" + maxLevel + ")"    AS startNodeTitle
+  WITH *, target.name   + "\\n(level " + target.maxDistanceFromSource + "/" + maxLevel + ")"  AS endNodeTitle
+               // The longest path will be highlighted in red.
+  WITH *, CASE WHEN isPartOfLongestPath       THEN "; color=\"red\"" 
+               // Dependencies contributing to the longest path will be highlighted in dark orange.
+               WHEN contributesToALongestPath THEN "; color=\"darkorange\"" 
+          ELSE "" END AS edgeColor
+// Prepare the GraphViz edge attributes for the visualization
+  WITH *, "[label=" + weight  + "; penwidth=" + penWidth + edgeColor + "; ];" AS graphVizEdgeAttributes
+// Assemble the final GraphViz DOT notation line for the edge representing the current dependency
+  WITH *, "\"" + startNodeTitle +  "\" -> \"" + endNodeTitle + "\" " + graphVizEdgeAttributes AS graphVizDotNotationLine
+RETURN DISTINCT graphVizDotNotationLine
+// Debugging
+//      ,source.name
+//      ,target.name
+//      ,numberOfLongestPathsPassing
+//      ,lengthOfLongestPathPassing
+//      ,contributesToALongestPath
+//      ,isPartOfLongestPath
+ LIMIT 440

cypher/Path_Finding/Path_Finding_6_Longest_paths_for_graphviz.cypher

@@ -0,0 +1,35 @@
+// Path Finding - Longest path - Stream - Find the top 100 dependencies contributing to the longest paths for Visualization with GraphViz
+
+ MATCH (sourceNodeForStatistics)-[dependencyForStatistics:DEPENDS_ON]->(targetNodeForStatistics)
+ WHERE $dependencies_projection_node IN LABELS(sourceNodeForStatistics)
+   AND $dependencies_projection_node IN LABELS(targetNodeForStatistics)
+  WITH min(dependencyForStatistics[$dependencies_projection_weight_property]) AS minWeight
+      ,max(dependencyForStatistics[$dependencies_projection_weight_property]) AS maxWeight
+      ,max(targetNodeForStatistics.maxDistanceFromSource)                     AS maxLevel
+   WITH *, 1.0 / toFloat(maxWeight - minWeight)  AS weightNormalizationFactor
+  CALL gds.dag.longestPath.stream($dependencies_projection + '-cleaned')
+ YIELD index, totalCost, path
+  WITH *, toInteger(totalCost) AS distance
+ ORDER BY distance DESC, index ASC
+ UNWIND relationships(path) AS pathRelationship
+  WITH *
+      ,startNode(pathRelationship) AS startNode
+      ,endNode(pathRelationship)   AS endNode
+ MATCH (startNode)-[dependency:DEPENDS_ON]->(endNode)
+  WITH *, dependency[$dependencies_projection_weight_property] AS weight
+  WITH *, toFloat(weight - minWeight) * weightNormalizationFactor AS normalizedWeight
+  WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth
+  WITH *, startNode.name + "\\n(level " + startNode.maxDistanceFromSource + "/" + maxLevel + ")" AS startNodeTitle
+  WITH *, endNode.name   + "\\n(level " + endNode.maxDistanceFromSource + "/" + maxLevel + ")"   AS endNodeTitle
+  WITH *, "[label=" + weight  + "; penwidth=" + penWidth + "; ];"    AS graphVizEdgeAttributes
+  WITH *, "\"" + startNodeTitle +  "\" -> \"" + endNodeTitle + "\" " + graphVizEdgeAttributes    AS graphVizDotNotationLine
+RETURN graphVizDotNotationLine
+// Debugging
+// RETURN startNode.name        AS startNodeName
+//       ,endNode.name          AS endNodeName
+//       ,dependency[$dependencies_projection_weight_property] AS dependencyWeight
+//       ,max(distance)         AS partOfLongestPathLength
+//       ,count(DISTINCT index) AS partOfLongestPathCounts
+//       ,startNode.maxDistanceFromSource AS startNodeLevel
+//       ,endNode.maxDistanceFromSource   AS endNodeLevel
+LIMIT 100

cypher/Path_Finding/Set_Parameters.cypher

@@ -1,4 +1,4 @@
-// Example on how to set the parameters for centrality in this case for Packages and PageRank
+// Example on how to set the parameters for path finding in this case for Packages and PageRank
 
 :params {
     "dependencies_projection": "package-path-finding",     

cypher/Topological_Sort/Set_Parameters.cypher

@@ -0,0 +1,7 @@
+// Example on how to set the parameters for topological sort in this case for Java Artifacts and Node Similarity
+
+:params {
+    "dependencies_projection": "artifact-topology",     
+    "dependencies_projection_node": "Artifact",
+    "dependencies_projection_weight_property": "weight"
+}

graph-visualization/DEPRECATED.md

@@ -0,0 +1,5 @@
+# Render Graph Visualizations (Deprecated)
+
+:warning: **This package is deprecated and might get removed in future.** :warning:
+
+Visualizations are now done using [GraphViz](https://graphviz.org). Please use the new script [visualizeQueryResults.sh](./../scripts/visualization/visualizeQueryResults.sh) like it is done in [InternalDependenciesVisualization.sh](./../scripts/reports/InternalDependenciesVisualization.sh) to create Graph Visualizations.

graph-visualization/README.md

@@ -1,6 +1,11 @@
-# Render Graph Visualizations
+# Render Graph Visualizations (Deprecated)
 
-This [node.js](https://nodejs.org/de) project provides the script [renderVisualizations.js](./renderVisualizations.js) to render all graph visualizations as image files.  
+:warning: **This package is deprecated and might get removed in future.** :warning:
+
+Visualizations are now done using [GraphViz](https://graphviz.org). Please use the new script [visualizeQueryResults.sh](./../scripts/visualization/visualizeQueryResults.sh) like it is done in [InternalDependenciesVisualization.sh](./../scripts/reports/InternalDependenciesVisualization.sh) to create Graph Visualizations.
+
+This [node.js](https://nodejs.org/de) project provides the script [renderVisualizations.js](./renderVisualizations.js) to render  graph visualizations as image files. It shows how to use [Puppeteer](https://pptr.dev) to render HTML5 Canvas elements as images,
+jimp to manipulate images and neovis to visualize Neo4j Graphs.
 
 ## Prerequisites
 

scripts/reports/GraphVisualization.sh

@@ -1,13 +1,16 @@
 #!/usr/bin/env bash
 
-# Creates the "graph-visualization" report (ipynb, md, pdf) based on the Jupyter Notebook "ArtifactDependencies.ipynb".
+# DEPRECATED: Creates the "graph-visualization" report using the (now deprecated) nodejs project "graph-visualization".
 # It contains the hierarchical artifact dependencies graph
 
 # Requires executeJupyterNotebook.sh
 
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+echo "GraphVisualization: Graph Visualization with the nodejs project 'graph-visualization' is deprecated and will be skipped."
+return 0
+
 # Overrideable Constants (defaults also defined in sub scripts)
 REPORTS_DIRECTORY=${REPORTS_DIRECTORY:-"reports"}