Merge pull request #319 from JohT/feature/improve-logging

JohT · web-flow · commit ea1677cc69a2 · 2025-01-31T12:25:38.000+01:00
Improve Logging
diff --git a/cypher/Dependencies_Projection/Dependencies_4c_Create_Undirected_Java_Type_Projection.cypher b/cypher/Dependencies_Projection/Dependencies_4c_Create_Undirected_Java_Type_Projection.cypher
@@ -1,6 +1,6 @@
-// Create filtered Type node projection without zero-degree nodes, external types, java types or duplicates. Variables: dependencies_projection. Requires 'Label_base_java_types', 'Label_buildin_java_types' and 'Label_resolved_duplicate_types' of 'Types' directory.
+// Create filtered Java Type node projection without zero-degree nodes, external types, java types or duplicates. Variables: dependencies_projection. Requires 'Label_base_java_types', 'Label_buildin_java_types' and 'Label_resolved_duplicate_types' of 'Types' directory.
 
-  MATCH (internalType:Type&!PrimitiveType&!Void&!JavaType&!ResolvedDuplicateType&!ExternalType)
+  MATCH (internalType:Java&Type&!PrimitiveType&!Void&!JavaType&!ResolvedDuplicateType&!ExternalType)
  OPTIONAL MATCH (internalType)-[typeDependency:DEPENDS_ON]->(dependentType:Type&!PrimitiveType&!Void&!JavaType&!ResolvedDuplicateType&!ExternalType)
   WITH internalType
       ,typeDependency
diff --git a/scripts/analysis/analyze.sh b/scripts/analysis/analyze.sh
@@ -39,6 +39,8 @@ REPORT_COMPILATIONS_SCRIPTS_DIRECTORY=${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY:-"
 SETTINGS_PROFILE_SCRIPTS_DIRECTORY=${SETTINGS_PROFILE_SCRIPTS_DIRECTORY:-"profiles"} # Repository directory that contains scripts containing settings
 ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"} # Working directory containing the artifacts to be analyzed
 SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"}
+LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command.
+LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command.
 
 # Function to display script usage
 usage() {
@@ -127,15 +129,21 @@ echo "analyze: Using analysis settings profile script ${SETTINGS_PROFILE_SCRIPT}
 source "${SETTINGS_PROFILE_SCRIPT}"
 
 # Setup Tools
+echo "${LOG_GROUP_START}Setup Tools";
 source "${SCRIPTS_DIR}/setupNeo4j.sh"
 source "${SCRIPTS_DIR}/setupJQAssistant.sh"
 source "${SCRIPTS_DIR}/startNeo4j.sh"
+echo "${LOG_GROUP_END}";
 
 # Scan and analyze artifacts when they were changed
+echo "${LOG_GROUP_START}Scan and Analyze Changed Artifacts";
 source "${SCRIPTS_DIR}/resetAndScanChanged.sh"
+echo "${LOG_GROUP_END}";
 
 # Prepare and validate graph database before analyzing and creating reports 
+echo "${LOG_GROUP_START}Prepare Analysis";
 source "${SCRIPTS_DIR}/prepareAnalysis.sh"
+echo "${LOG_GROUP_END}";
 
 if ${exploreMode}; then
   echo "analyze: Explore mode activated. Report generation will be skipped. Neo4j keeps running."
@@ -149,4 +157,6 @@ echo "analyze: Creating Reports with ${REPORT_COMPILATION_SCRIPT} ..."
 source "${REPORT_COMPILATION_SCRIPT}"
 
 # Stop Neo4j at the end
-source "${SCRIPTS_DIR}/stopNeo4j.sh"
+echo "${LOG_GROUP_START}Finishing Analysis";
+source "${SCRIPTS_DIR}/stopNeo4j.sh"
+echo "${LOG_GROUP_END}";
diff --git a/scripts/executeQuery.sh b/scripts/executeQuery.sh
@@ -21,6 +21,10 @@ set -o errexit -o pipefail
 NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} # Neo4j HTTP API port for executing queries
 NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/neo4j/tx/commit"} # Neo4j v5: "db/<name>/tx/commit", Neo4j v4: "db/data/transaction/commit"
 
+# Local constants
+ERROR_COLOR='\033[0;31m'
+NO_COLOR='\033[0m'
+
 # Check if environment variable is set
 if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then
     echo "executeQuery requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD=<your-own-password>'." >&2
@@ -30,13 +34,15 @@ fi
 # Input Arguments: Initialize arguments and set default values for optional ones
 cypher_query_file_name=""
 no_source_reference=false
+omit_query_error_highlighting=false
 query_parameters=""
 
 # Input Arguments: Function to print usage information
 print_usage() {
-    echo "executeQuery Usage: $0 <filename> [--no-source-reference-column]" >&2
+    echo "executeQuery Usage: $0 <filename> [--no-source-reference-column] [--omit-query-error-highlighting]" >&2
     echo "Options:" >&2
     echo "  --no-source-reference-column: Exclude the source reference column" >&2
+    echo "  --omit-query-error-highlighting: Log query errors in same color as infos" >&2
 }
 
 # Input Arguments: Parse the command-line arguments
@@ -48,6 +54,10 @@ while [[ $# -gt 0 ]]; do
             no_source_reference=true
             shift
             ;;
+        --omit_query_error_highlighting)
+            omit_query_error_highlighting=true
+            shift
+            ;;
         *)
             if [[ -z "${cypher_query_file_name}" ]]; then
                 # Input Arguments: Read the first unnamed input argument containing the name of the cypher file
@@ -78,6 +88,13 @@ done
 
 #echo "executeQuery: query_parameters: ${query_parameters}"
 
+# Set the color for error messages
+error_message_color="${ERROR_COLOR}"
+if [ "${omit_query_error_highlighting}" = "true" ] ; then
+  error_message_color="${NO_COLOR}"
+  echo "executeQuery: Ommiting error highlighting" >&2
+fi
+
 # Read the file that contains the Cypher query
 original_cypher_query=$(<"${cypher_query_file_name}")
 #echo "executeQuery: Original Query: $original_cypher_query"
@@ -102,21 +119,18 @@ if ! cypher_query_result=$(curl --silent -S --fail-with-body -H Accept:applicati
      "http://localhost:${NEO4J_HTTP_PORT}/${NEO4J_HTTP_TRANSACTION_ENDPOINT}" \
      -d "${cypher_query_for_api}" 2>&1) ;
 then
-  redColor='\033[0;31m'
-  noColor='\033[0m'
-  echo -e "${redColor}${cypher_query_file_name}: ${cypher_query_result}${noColor}" >&2
-  echo -e "${redColor}Parameters: ${query_parameters}${noColor}" >&2
+  echo -e "${error_message_color}${cypher_query_file_name}: ${cypher_query_result}${NO_COLOR}" >&2
+  echo -e "${error_message_color}Parameters: ${query_parameters}${NO_COLOR}" >&2
   exit 1
 fi
 #echo "executeQuery: Cypher Query OK Result: ${cypher_query_result}"
 
-# If there is a error message print it to syserr >&2 in red color
+# If there is a error message print it to syserr >&2 in error color
 error_message=$( echo "${cypher_query_result}" | jq -r '.errors[0] // empty' )
-if [[ -n "${error_message}" ]]; then 
-  redColor='\033[0;31m'
-  noColor='\033[0m'
-  echo -e "${redColor}${cypher_query_file_name}: ${error_message}${noColor}" >&2
-  echo -e "${redColor}Parameters: ${query_parameters}${noColor}" >&2
+if [[ -n "${error_message}" ]]; then
+  # Set the message color to red if the query errors should be highlighted
+  echo -e "${error_message_color}${cypher_query_file_name}: ${error_message}${NO_COLOR}" >&2
+  echo -e "${error_message_color}Parameters: ${query_parameters}${NO_COLOR}" >&2
   exit 1
 fi
 
diff --git a/scripts/projectionFunctions.sh b/scripts/projectionFunctions.sh
@@ -238,7 +238,7 @@ createUndirectedJavaTypeDependencyProjection() {
     execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_2_Delete_Subgraph.cypher" "${@}" >/dev/null
 
     local projectionResult
-    projectionResult=$( execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_4c_Create_Undirected_Type_Projection.cypher" "${@}")
+    projectionResult=$( execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_4c_Create_Undirected_Java_Type_Projection.cypher" "${@}")
     if is_csv_column_greater_zero "${projectionResult}" "relationshipCount"; then
         true;
     else
diff --git a/scripts/reports/CentralityCsv.sh b/scripts/reports/CentralityCsv.sh
@@ -407,7 +407,7 @@ TYPE_PROJECTION_UNDIRECTED="dependencies_projection=type-centrality-undirected"
 TYPE_NODE="dependencies_projection_node=Type" 
 TYPE_WEIGHT="dependencies_projection_weight_property=weight" 
 
-if createDirectedJavaTypeDependencyProjection "${TYPE_PROJECTION}" "${TYPE_NODE}" "${TYPE_WEIGHT}"; then
+if createDirectedJavaTypeDependencyProjection "${TYPE_PROJECTION}"; then
     runCentralityAlgorithms "${TYPE_PROJECTION}" "${TYPE_NODE}" "${TYPE_WEIGHT}"
 fi
 if createUndirectedJavaTypeDependencyProjection "${TYPE_PROJECTION_UNDIRECTED}"; then
diff --git a/scripts/reports/compilations/CsvReports.sh b/scripts/reports/compilations/CsvReports.sh
@@ -8,6 +8,10 @@
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+# Overrideable Constants (defaults also defined in sub scripts)
+LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command.
+LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command.
+
 ## Get this "scripts/reports/compilations" directory if not already set.
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -19,7 +23,14 @@ REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCR
 echo "CsvReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}"
 
 # Run all report scripts
-for report_script_file in "${REPORTS_SCRIPT_DIR}"/*Csv.sh; do 
-    echo "CsvReports: Starting ${report_script_file}..."; 
+for report_script_file in "${REPORTS_SCRIPT_DIR}"/*Csv.sh; do
+    report_script_filename=$(basename -- "${report_script_file}")
+    
+    echo "${LOG_GROUP_START}${report_script_file}";
+    echo "CsvReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${report_script_filename}...";
+
     source "${report_script_file}"
-done
+
+    echo "CsvReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${report_script_filename}";
+    echo "${LOG_GROUP_END}";
+done
diff --git a/scripts/reports/compilations/JupyterReports.sh b/scripts/reports/compilations/JupyterReports.sh
@@ -11,6 +11,10 @@
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+# Overrideable Constants (defaults also defined in sub scripts)
+LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command.
+LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command.
+
 ## Get this "scripts/reports/compilations" directory if not already set.
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -29,9 +33,15 @@ echo "JupyterReports: SCRIPTS_DIR=${SCRIPTS_DIR}"
 JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY:-"${SCRIPTS_DIR}/../jupyter"} # Repository directory containing the Jupyter Notebooks
 echo "JupyterReports: JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY}"
 
-# Run all report scripts
+# Run all jupiter notebooks
 for jupyter_notebook_file in "${JUPYTER_NOTEBOOK_DIRECTORY}"/*.ipynb; do 
-    jupyter_notebook_file=$( basename "${jupyter_notebook_file}")
-    echo "JupyterReports: Executing ${jupyter_notebook_file}..."; 
-    source "${SCRIPTS_DIR}/executeJupyterNotebookReport.sh" --jupyterNotebook "${jupyter_notebook_file}"
+    jupyter_notebook_filename=$(basename -- "${jupyter_notebook_file}")
+    
+    echo "${LOG_GROUP_START}${jupyter_notebook_filename}";
+    echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${jupyter_notebook_filename}...";
+
+    source "${SCRIPTS_DIR}/executeJupyterNotebookReport.sh" --jupyterNotebook "${jupyter_notebook_filename}"
+
+    echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${jupyter_notebook_filename}";
+    echo "${LOG_GROUP_END}";
 done
diff --git a/scripts/reports/compilations/VisualizationReports.sh b/scripts/reports/compilations/VisualizationReports.sh
@@ -11,6 +11,10 @@
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+# Overrideable Constants (defaults also defined in sub scripts)
+LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command.
+LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command.
+
 ## Get this "scripts/reports/compilations" directory if not already set.
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -21,8 +25,15 @@ echo "VisualizationReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS
 REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCRIPT_DIR}")}
 echo "VisualizationReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}"
 
-# Run all report scripts
-for report_script_file in "${REPORTS_SCRIPT_DIR}"/*Visualization.sh; do 
-    echo "VisualizationReports: Starting ${report_script_file}..."; 
-    source "${report_script_file}"
+# Run all visualization scripts
+for visualization_script_file in "${REPORTS_SCRIPT_DIR}"/*Visualization.sh; do 
+    visualization_script_filename=$(basename -- "${visualization_script_file}")
+    
+    echo "${LOG_GROUP_START}${visualization_script_filename}";
+    echo "VisualizationReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${visualization_script_filename}...";
+
+    source "${visualization_script_file}"
+
+    echo "VisualizationReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${visualization_script_filename}";
+    echo "${LOG_GROUP_END}";
 done
diff --git a/scripts/waitForNeo4jHttpFunctions.sh b/scripts/waitForNeo4jHttpFunctions.sh
@@ -24,13 +24,13 @@ echo "waitForNeo4jHttp: CYPHER_DIR=${CYPHER_DIR}"
 # Define functions to execute a Cypher query from within the given file (first and only argument)
 source "${SCRIPTS_DIR}/executeQueryFunctions.sh"
 
-queryDatabase() { 
-    execute_cypher "${CYPHER_DIR}/Count_nodes_and_relationships.cypher" "--no-source-reference-column"
+testDatabase() { 
+    execute_cypher "${CYPHER_DIR}/Count_nodes_and_relationships.cypher" "--no-source-reference-column" "--omit_query_error_highlighting"
 }
 
 isDatabaseQueryable() { 
     local cypher_elements_query_result
-    if cypher_elements_query_result=$(queryDatabase);
+    if cypher_elements_query_result=$(testDatabase);
     then
         echo "true"
      else
@@ -56,7 +56,7 @@ waitUntilDatabaseIsQueryable() {
             continue; # query failed -> try again
         fi
         
-        local queryResult=$(queryDatabase || true)
+        local queryResult=$(testDatabase || true)
         if [[ -n "${queryResult}" ]]; then 
             echo "waitForNeo4jHttp: Successfully accessed Neo4j HTTP API."
             echo "${queryResult}"
