Merge pull request #54 from JohT/feature/fail-fast-error-handling

Improve error handling using a fail-fast approach

Author: JohT

.github/workflows/code-structure-analysis.yml

@@ -121,6 +121,7 @@ jobs:
       shell: bash -el {0}
       env:
         NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }}
+        ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: "true"
       run: |
         ./../../scripts/analysis/analyze.sh
 
@@ -136,7 +137,6 @@ jobs:
         name: code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
         path: |
           ./temp/**/runtime/*
-          ./results
         retention-days: 5
 
     # Upload successful results in case they are needed for troubleshooting

COMMANDS.md

@@ -83,12 +83,12 @@ If only the Jupyter reports are needed e.g. when the CSV reports had already bee
 ./../../scripts/analysis/analyze.sh --report Jupyter
 ```
 
-#### Start an analysis without PDF generation
+#### Start an analysis with PDF generation
 
-Generating a PDF from a Jupyter notebook using [nbconvert](https://nbconvert.readthedocs.io) might take a while or even fail due to a timeout error. Here is an example on how to skip PDF generation:
+Note: Generating a PDF from a Jupyter notebook using [nbconvert](https://nbconvert.readthedocs.io) takes some time and might even fail due to a timeout error.
 
 ```shell
-SKIP_JUPYTER_NOTEBOOK_PDF_GENERATION=true ./../../scripts/analysis/analyze.sh
+ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION=true ./../../scripts/analysis/analyze.sh
 ```
 
 #### Setup everything to explore the graph manually

README.md

@@ -124,15 +124,15 @@ The [Code Structure Analysis Pipeline](./.github/workflows/code-structure-analys
 - How can i trigger a full rescan of all artifacts?  
   👉 Delete the file `artifactsChangeDetectionHash.txt` in the `artifacts` directory.
 
-- How can PDF generation be skipped to speed up report generation and not depend on chromium?  
-  👉 Set environment variable `SKIP_JUPYTER_NOTEBOOK_PDF_GENERATION` to anything except an empty string. Example:  
+- How can PDF generation for Jupyter Notebooks be enabled (depends on chromium, takes more time)?  
+  👉 Set environment variable `ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION` to anything except an empty string. Example:  
 
   ```shell
-  export SKIP_JUPYTER_NOTEBOOK_PDF_GENERATION="true"
+  export ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION="true"
   ```
 
-  👉 Or prepend your command with `SKIP_JUPYTER_NOTEBOOK_PDF_GENERATION="true"` like:  
+  👉 Alternatively prepend your command with `ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION="true"` like:  
 
   ```shell
-  SKIP_JUPYTER_NOTEBOOK_PDF_GENERATION=true ./../../scripts/analysis/analyze.sh
+  ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION=true ./../../scripts/analysis/analyze.sh
   ```

cypher/Community_Detection/Community_Detection_7d_Modularity_Members.cypher

@@ -6,17 +6,25 @@ CALL gds.modularity.stream(
     ,communityProperty: $dependencies_projection_write_property
 })
  YIELD communityId, modularity
-  WITH communityId, modularity
+  WITH collect({communityId: communityId, modularity: modularity}) AS modularities
  MATCH (member)
- WHERE member[$dependencies_projection_write_property] = communityId
+ WHERE member[$dependencies_projection_write_property] IS NOT NULL
    AND $dependencies_projection_node IN LABELS(member)
-  WITH communityId
-      ,modularity
+  WITH modularities 
+      ,member[$dependencies_projection_write_property]                               AS communityId
       ,coalesce(member.fqn, member.fileName, member.name)                            AS memberName
       ,coalesce(member.name, replace(last(split(member.fileName, '/')), '.jar', '')) AS shortMemberName
-RETURN communityId
-      ,modularity
+  WITH modularities
+      ,communityId
       ,count(DISTINCT memberName)        AS memberCount
       ,collect(DISTINCT shortMemberName) AS shortMemberNames
       ,collect(DISTINCT memberName)      AS memberNames
+      ,reduce(memberModularity = 0, modularity IN modularities | 
+        CASE modularity.communityId WHEN communityId THEN modularity.modularity 
+        ELSE memberModularity END)       AS memberModularity
+ RETURN communityId
+       ,memberModularity
+       ,memberCount
+       ,shortMemberNames
+       ,memberNames
 ORDER BY communityId ASCENDING

cypher/Community_Detection/Community_Detection_7e_Write_Modularity.cypher

@@ -1,15 +1,20 @@
-// Community Detection Write Modularity
+// Community Detection Modularity Write
 
 CALL gds.modularity.stream(
  $dependencies_projection + '-without-empty', {
      relationshipWeightProperty: $dependencies_projection_weight_property
     ,communityProperty: $dependencies_projection_write_property
 })
  YIELD communityId, modularity
-  WITH communityId, modularity
+  WITH collect({communityId: communityId, modularity: modularity}) AS modularities
  MATCH (member)
- WHERE member[$dependencies_projection_write_property] = communityId
+ WHERE member[$dependencies_projection_write_property] IS NOT NULL
    AND $dependencies_projection_node IN LABELS(member)
-  CALL apoc.create.setProperty(member, $dependencies_projection_write_property + 'Modularity', modularity)
+  WITH modularities 
+      ,member
+      ,reduce(memberModularity = 0, modularity IN modularities | 
+        CASE modularity.communityId WHEN member[$dependencies_projection_write_property] THEN modularity.modularity 
+        ELSE memberModularity END)       AS memberModularity
+  CALL apoc.create.setProperty(member, $dependencies_projection_write_property + 'Modularity', memberModularity)
  YIELD node
-RETURN count(DISTINCT node) AS writtenModularityNodes;
+RETURN count(DISTINCT node) AS writtenModularityNodes

cypher/Community_Detection/Compare_Louvain_vs_Leiden_Results.cypher

@@ -2,12 +2,12 @@
 
  MATCH (member)
  WHERE $dependencies_projection_node IN LABELS(member)
+   AND member.louvainCommunityId IS NOT NULL
+   AND member.leidenCommunityId  IS NOT NULL
  WITH member.communityLouvainId    AS louvainCommunityId
      ,member.communityLeidenId     AS leidenCommunityId
      ,coalesce(member.fqn, member.fileName, member.signature, member.name)          AS memberName
      ,coalesce(member.name, replace(last(split(member.fileName, '/')), '.jar', '')) AS shortMemberName
- WHERE louvainCommunityId IS NOT NULL
-   AND leidenCommunityId  IS NOT NULL
  WITH louvainCommunityId
      ,leidenCommunityId
      ,collect(DISTINCT shortMemberName) AS shortMemberNames

scripts/analysis/analyze.sh

@@ -30,6 +30,9 @@
 
 # Requires setupNeo4j.sh,setupJQAssistant.sh,startNeo4j.sh,resetAndScanChanged.sh,prepareAnalysis.sh,stopNeo4j.sh,comilations/*.sh,profiles/*.sh
 
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -eo pipefail
+
 # Overrideable variables with directory names
 REPORTS_SCRIPTS_DIRECTORY=${REPORTS_SCRIPTS_DIRECTORY:-"reports"} # Working directory containing the generated reports
 REPORT_COMPILATIONS_SCRIPTS_DIRECTORY=${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY:-"compilations"} # Repository directory that contains scripts that execute selected report generation scripts
@@ -116,18 +119,18 @@ fi
 
 # Execute the settings profile script that sets all the neccessary settings variables (overrideable by environment variables).
 echo "analyze: Using analysis settings profile script ${SETTINGS_PROFILE_SCRIPT}"
-source "${SETTINGS_PROFILE_SCRIPT}" || exit 2
+source "${SETTINGS_PROFILE_SCRIPT}"
 
 # Setup Tools
-source "${SCRIPTS_DIR}/setupNeo4j.sh" || exit 3
-source "${SCRIPTS_DIR}/setupJQAssistant.sh" || exit 3
-source "${SCRIPTS_DIR}/startNeo4j.sh" || exit 3
+source "${SCRIPTS_DIR}/setupNeo4j.sh"
+source "${SCRIPTS_DIR}/setupJQAssistant.sh"
+source "${SCRIPTS_DIR}/startNeo4j.sh"
 
 # Scan and analyze artifacts when they were changed
-source "${SCRIPTS_DIR}/resetAndScanChanged.sh" || exit 4
+source "${SCRIPTS_DIR}/resetAndScanChanged.sh"
 
 # Prepare and validate graph database before analyzing and creating reports 
-source "${SCRIPTS_DIR}/prepareAnalysis.sh" || exit 5
+source "${SCRIPTS_DIR}/prepareAnalysis.sh"
 
 if ${exploreMode}; then
   echo "analyze: Explore mode activated. Report generation will be skipped. Neo4j keeps running."
@@ -138,7 +141,7 @@ fi
 # Create Reports
 #########################
 echo "analyze: Creating Reports with ${REPORT_COMPILATION_SCRIPT} ..."
-source "${REPORT_COMPILATION_SCRIPT}" || exit 6
+source "${REPORT_COMPILATION_SCRIPT}"
 
 # Stop Neo4j at the end
 source "${SCRIPTS_DIR}/stopNeo4j.sh"

scripts/copyReportsIntoResults.sh

@@ -8,6 +8,9 @@
 
 # Requires generateJupyterReportReference.sh
 
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -eo pipefail
+
 ## Get this "scripts" directory if not already set
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.

scripts/detectChangedArtifacts.sh

@@ -4,6 +4,9 @@
 # The hash value is generated based on all files (their names and properties) within the artifacts directory.
 # A change is detected when the current hash and the stored one differ.
 
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -eo pipefail
+
 ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"}
 ARTIFACTS_CHANGE_DETECTION_HASH_FILE=${ARTIFACTS_CHANGE_DETECTION_HASH_FILE:-"artifactsChangeDetectionHash.txt"} # Name of the file that contains the hash code of the file list for change detection
 ARTIFACTS_CHANGE_DETECTION_HASH_FILE_PATH="./${ARTIFACTS_DIRECTORY}/$ARTIFACTS_CHANGE_DETECTION_HASH_FILE"
@@ -16,7 +19,9 @@ fi
 
 # Use find to list all files in the directory with their properties,
 # sort the output, and pipe it to md5 to create a hash
-CURRENT_ARTIFACTS_HASH="$( find "./$ARTIFACTS_DIRECTORY" -type f -not -name "${ARTIFACTS_CHANGE_DETECTION_HASH_FILE}" -exec stat -f "%N %p %z" {} + | sort | md5 -r )"
+# Use openssl md5 that is at least available on Mac and Linux. 
+# See: https://github.com/TRON-US/go-btfs/issues/90#issuecomment-517409369
+CURRENT_ARTIFACTS_HASH="$( find "./$ARTIFACTS_DIRECTORY" -type f -not -name "${ARTIFACTS_CHANGE_DETECTION_HASH_FILE}" -exec openssl md5 -binary {} + | openssl md5 | awk '{print $2}' )"
 
 # Assume that the files where changed if the file containing the hash of the file list does not exist yet.
 if [ ! -f "${ARTIFACTS_CHANGE_DETECTION_HASH_FILE_PATH}" ] ; then

scripts/documentation/appendEnvironmentVariables.sh

@@ -5,6 +5,9 @@
 # Note: If called with "clear" instead of a filename then the generated markdown reference documentation file is deleted.
 #       This is helpful to start over with the generation of a new document.
 
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -eo pipefail
+
 # Markdown file name
 markdownFile="ENVIRONMENT_VARIABLES.md"