From 710c8f95b8739d188c10fb13090df5b7846cfbfb Mon Sep 17 00:00:00 2001
From: JohT <7671054+JohT@users.noreply.github.com>
Date: Wed, 29 Jan 2025 07:50:17 +0100
Subject: [PATCH 1/3] Omit coloring expected query error message

---
 scripts/executeQuery.sh              | 36 +++++++++++++++++++---------
 scripts/waitForNeo4jHttpFunctions.sh |  8 +++----
 2 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/scripts/executeQuery.sh b/scripts/executeQuery.sh
index e9899c60e..b8d7cf4e3 100755
--- a/scripts/executeQuery.sh
+++ b/scripts/executeQuery.sh
@@ -21,6 +21,10 @@ set -o errexit -o pipefail
 NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} # Neo4j HTTP API port for executing queries
 NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/neo4j/tx/commit"} # Neo4j v5: "db/<name>/tx/commit", Neo4j v4: "db/data/transaction/commit"
 
+# Local constants
+ERROR_COLOR='\033[0;31m'
+NO_COLOR='\033[0m'
+
 # Check if environment variable is set
 if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then
     echo "executeQuery requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD=<your-own-password>'." >&2
@@ -30,13 +34,15 @@ fi
 # Input Arguments: Initialize arguments and set default values for optional ones
 cypher_query_file_name=""
 no_source_reference=false
+omit_query_error_highlighting=false
 query_parameters=""
 
 # Input Arguments: Function to print usage information
 print_usage() {
-    echo "executeQuery Usage: $0 <filename> [--no-source-reference-column]" >&2
+    echo "executeQuery Usage: $0 <filename> [--no-source-reference-column] [--omit-query-error-highlighting]" >&2
     echo "Options:" >&2
     echo "  --no-source-reference-column: Exclude the source reference column" >&2
+    echo "  --omit-query-error-highlighting: Log query errors in same color as infos" >&2
 }
 
 # Input Arguments: Parse the command-line arguments
@@ -48,6 +54,10 @@ while [[ $# -gt 0 ]]; do
             no_source_reference=true
             shift
             ;;
+        --omit_query_error_highlighting)
+            omit_query_error_highlighting=true
+            shift
+            ;;
         *)
             if [[ -z "${cypher_query_file_name}" ]]; then
                 # Input Arguments: Read the first unnamed input argument containing the name of the cypher file
@@ -78,6 +88,13 @@ done
 
 #echo "executeQuery: query_parameters: ${query_parameters}"
 
+# Set the color for error messages
+error_message_color="${ERROR_COLOR}"
+if [ "${omit_query_error_highlighting}" = "true" ] ; then
+  error_message_color="${NO_COLOR}"
+  echo "executeQuery: Ommiting error highlighting" >&2
+fi
+
 # Read the file that contains the Cypher query
 original_cypher_query=$(<"${cypher_query_file_name}")
 #echo "executeQuery: Original Query: $original_cypher_query"
@@ -102,21 +119,18 @@ if ! cypher_query_result=$(curl --silent -S --fail-with-body -H Accept:applicati
      "http://localhost:${NEO4J_HTTP_PORT}/${NEO4J_HTTP_TRANSACTION_ENDPOINT}" \
      -d "${cypher_query_for_api}" 2>&1) ;
 then
-  redColor='\033[0;31m'
-  noColor='\033[0m'
-  echo -e "${redColor}${cypher_query_file_name}: ${cypher_query_result}${noColor}" >&2
-  echo -e "${redColor}Parameters: ${query_parameters}${noColor}" >&2
+  echo -e "${error_message_color}${cypher_query_file_name}: ${cypher_query_result}${NO_COLOR}" >&2
+  echo -e "${error_message_color}Parameters: ${query_parameters}${NO_COLOR}" >&2
   exit 1
 fi
 #echo "executeQuery: Cypher Query OK Result: ${cypher_query_result}"
 
-# If there is a error message print it to syserr >&2 in red color
+# If there is a error message print it to syserr >&2 in error color
 error_message=$( echo "${cypher_query_result}" | jq -r '.errors[0] // empty' )
-if [[ -n "${error_message}" ]]; then 
-  redColor='\033[0;31m'
-  noColor='\033[0m'
-  echo -e "${redColor}${cypher_query_file_name}: ${error_message}${noColor}" >&2
-  echo -e "${redColor}Parameters: ${query_parameters}${noColor}" >&2
+if [[ -n "${error_message}" ]]; then
+  # Set the message color to red if the query errors should be highlighted
+  echo -e "${error_message_color}${cypher_query_file_name}: ${error_message}${NO_COLOR}" >&2
+  echo -e "${error_message_color}Parameters: ${query_parameters}${NO_COLOR}" >&2
   exit 1
 fi
 
diff --git a/scripts/waitForNeo4jHttpFunctions.sh b/scripts/waitForNeo4jHttpFunctions.sh
index dd40f7c3b..ec01192c8 100644
--- a/scripts/waitForNeo4jHttpFunctions.sh
+++ b/scripts/waitForNeo4jHttpFunctions.sh
@@ -24,13 +24,13 @@ echo "waitForNeo4jHttp: CYPHER_DIR=${CYPHER_DIR}"
 # Define functions to execute a Cypher query from within the given file (first and only argument)
 source "${SCRIPTS_DIR}/executeQueryFunctions.sh"
 
-queryDatabase() { 
-    execute_cypher "${CYPHER_DIR}/Count_nodes_and_relationships.cypher" "--no-source-reference-column"
+testDatabase() { 
+    execute_cypher "${CYPHER_DIR}/Count_nodes_and_relationships.cypher" "--no-source-reference-column" "--omit_query_error_highlighting"
 }
 
 isDatabaseQueryable() { 
     local cypher_elements_query_result
-    if cypher_elements_query_result=$(queryDatabase);
+    if cypher_elements_query_result=$(testDatabase);
     then
         echo "true"
      else
@@ -56,7 +56,7 @@ waitUntilDatabaseIsQueryable() {
             continue; # query failed -> try again
         fi
         
-        local queryResult=$(queryDatabase || true)
+        local queryResult=$(testDatabase || true)
         if [[ -n "${queryResult}" ]]; then 
             echo "waitForNeo4jHttp: Successfully accessed Neo4j HTTP API."
             echo "${queryResult}"

From 085786fea76e2b27781394d6f8c3e2926ca751d3 Mon Sep 17 00:00:00 2001
From: JohT <7671054+JohT@users.noreply.github.com>
Date: Wed, 29 Jan 2025 08:01:46 +0100
Subject: [PATCH 2/3] Emphasize Java language dependent projection creation

---
 ...ndencies_4c_Create_Undirected_Java_Type_Projection.cypher} | 4 ++--
 scripts/projectionFunctions.sh                                | 2 +-
 scripts/reports/CentralityCsv.sh                              | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
 rename cypher/Dependencies_Projection/{Dependencies_4c_Create_Undirected_Type_Projection.cypher => Dependencies_4c_Create_Undirected_Java_Type_Projection.cypher} (73%)

diff --git a/cypher/Dependencies_Projection/Dependencies_4c_Create_Undirected_Type_Projection.cypher b/cypher/Dependencies_Projection/Dependencies_4c_Create_Undirected_Java_Type_Projection.cypher
similarity index 73%
rename from cypher/Dependencies_Projection/Dependencies_4c_Create_Undirected_Type_Projection.cypher
rename to cypher/Dependencies_Projection/Dependencies_4c_Create_Undirected_Java_Type_Projection.cypher
index 342a06ffc..6c1d5fbdc 100644
--- a/cypher/Dependencies_Projection/Dependencies_4c_Create_Undirected_Type_Projection.cypher
+++ b/cypher/Dependencies_Projection/Dependencies_4c_Create_Undirected_Java_Type_Projection.cypher
@@ -1,6 +1,6 @@
-// Create filtered Type node projection without zero-degree nodes, external types, java types or duplicates. Variables: dependencies_projection. Requires 'Label_base_java_types', 'Label_buildin_java_types' and 'Label_resolved_duplicate_types' of 'Types' directory.
+// Create filtered Java Type node projection without zero-degree nodes, external types, java types or duplicates. Variables: dependencies_projection. Requires 'Label_base_java_types', 'Label_buildin_java_types' and 'Label_resolved_duplicate_types' of 'Types' directory.
 
-  MATCH (internalType:Type&!PrimitiveType&!Void&!JavaType&!ResolvedDuplicateType&!ExternalType)
+  MATCH (internalType:Java&Type&!PrimitiveType&!Void&!JavaType&!ResolvedDuplicateType&!ExternalType)
  OPTIONAL MATCH (internalType)-[typeDependency:DEPENDS_ON]->(dependentType:Type&!PrimitiveType&!Void&!JavaType&!ResolvedDuplicateType&!ExternalType)
   WITH internalType
       ,typeDependency
diff --git a/scripts/projectionFunctions.sh b/scripts/projectionFunctions.sh
index 7f745ade6..5a2af2b0d 100644
--- a/scripts/projectionFunctions.sh
+++ b/scripts/projectionFunctions.sh
@@ -238,7 +238,7 @@ createUndirectedJavaTypeDependencyProjection() {
     execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_2_Delete_Subgraph.cypher" "${@}" >/dev/null
 
     local projectionResult
-    projectionResult=$( execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_4c_Create_Undirected_Type_Projection.cypher" "${@}")
+    projectionResult=$( execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_4c_Create_Undirected_Java_Type_Projection.cypher" "${@}")
     if is_csv_column_greater_zero "${projectionResult}" "relationshipCount"; then
         true;
     else
diff --git a/scripts/reports/CentralityCsv.sh b/scripts/reports/CentralityCsv.sh
index 161a30ac3..7c74864ea 100755
--- a/scripts/reports/CentralityCsv.sh
+++ b/scripts/reports/CentralityCsv.sh
@@ -407,7 +407,7 @@ TYPE_PROJECTION_UNDIRECTED="dependencies_projection=type-centrality-undirected"
 TYPE_NODE="dependencies_projection_node=Type" 
 TYPE_WEIGHT="dependencies_projection_weight_property=weight" 
 
-if createDirectedJavaTypeDependencyProjection "${TYPE_PROJECTION}" "${TYPE_NODE}" "${TYPE_WEIGHT}"; then
+if createDirectedJavaTypeDependencyProjection "${TYPE_PROJECTION}"; then
     runCentralityAlgorithms "${TYPE_PROJECTION}" "${TYPE_NODE}" "${TYPE_WEIGHT}"
 fi
 if createUndirectedJavaTypeDependencyProjection "${TYPE_PROJECTION_UNDIRECTED}"; then

From 7d97d6541968e35e10ee754ab94157fd0c218596 Mon Sep 17 00:00:00 2001
From: JohT <7671054+JohT@users.noreply.github.com>
Date: Fri, 31 Jan 2025 07:59:14 +0100
Subject: [PATCH 3/3] Introduce log groups for GitHub Actions

---
 scripts/analysis/analyze.sh                   | 12 +++++++++++-
 scripts/reports/compilations/CsvReports.sh    | 17 ++++++++++++++---
 .../reports/compilations/JupyterReports.sh    | 18 ++++++++++++++----
 .../compilations/VisualizationReports.sh      | 19 +++++++++++++++----
 4 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/scripts/analysis/analyze.sh b/scripts/analysis/analyze.sh
index ba02f4e63..302e9c347 100755
--- a/scripts/analysis/analyze.sh
+++ b/scripts/analysis/analyze.sh
@@ -39,6 +39,8 @@ REPORT_COMPILATIONS_SCRIPTS_DIRECTORY=${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY:-"
 SETTINGS_PROFILE_SCRIPTS_DIRECTORY=${SETTINGS_PROFILE_SCRIPTS_DIRECTORY:-"profiles"} # Repository directory that contains scripts containing settings
 ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"} # Working directory containing the artifacts to be analyzed
 SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"}
+LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command.
+LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command.
 
 # Function to display script usage
 usage() {
@@ -127,15 +129,21 @@ echo "analyze: Using analysis settings profile script ${SETTINGS_PROFILE_SCRIPT}
 source "${SETTINGS_PROFILE_SCRIPT}"
 
 # Setup Tools
+echo "${LOG_GROUP_START}Setup Tools";
 source "${SCRIPTS_DIR}/setupNeo4j.sh"
 source "${SCRIPTS_DIR}/setupJQAssistant.sh"
 source "${SCRIPTS_DIR}/startNeo4j.sh"
+echo "${LOG_GROUP_END}";
 
 # Scan and analyze artifacts when they were changed
+echo "${LOG_GROUP_START}Scan and Analyze Changed Artifacts";
 source "${SCRIPTS_DIR}/resetAndScanChanged.sh"
+echo "${LOG_GROUP_END}";
 
 # Prepare and validate graph database before analyzing and creating reports 
+echo "${LOG_GROUP_START}Prepare Analysis";
 source "${SCRIPTS_DIR}/prepareAnalysis.sh"
+echo "${LOG_GROUP_END}";
 
 if ${exploreMode}; then
   echo "analyze: Explore mode activated. Report generation will be skipped. Neo4j keeps running."
@@ -149,4 +157,6 @@ echo "analyze: Creating Reports with ${REPORT_COMPILATION_SCRIPT} ..."
 source "${REPORT_COMPILATION_SCRIPT}"
 
 # Stop Neo4j at the end
-source "${SCRIPTS_DIR}/stopNeo4j.sh"
\ No newline at end of file
+echo "${LOG_GROUP_START}Finishing Analysis";
+source "${SCRIPTS_DIR}/stopNeo4j.sh"
+echo "${LOG_GROUP_END}";
\ No newline at end of file
diff --git a/scripts/reports/compilations/CsvReports.sh b/scripts/reports/compilations/CsvReports.sh
index 536f57cbf..a67d49ad9 100755
--- a/scripts/reports/compilations/CsvReports.sh
+++ b/scripts/reports/compilations/CsvReports.sh
@@ -8,6 +8,10 @@
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+# Overrideable Constants (defaults also defined in sub scripts)
+LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command.
+LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command.
+
 ## Get this "scripts/reports/compilations" directory if not already set.
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -19,7 +23,14 @@ REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCR
 echo "CsvReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}"
 
 # Run all report scripts
-for report_script_file in "${REPORTS_SCRIPT_DIR}"/*Csv.sh; do 
-    echo "CsvReports: Starting ${report_script_file}..."; 
+for report_script_file in "${REPORTS_SCRIPT_DIR}"/*Csv.sh; do
+    report_script_filename=$(basename -- "${report_script_file}")
+    
+    echo "${LOG_GROUP_START}${report_script_file}";
+    echo "CsvReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${report_script_filename}...";
+
     source "${report_script_file}"
-done
+
+    echo "CsvReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${report_script_filename}";
+    echo "${LOG_GROUP_END}";
+done
\ No newline at end of file
diff --git a/scripts/reports/compilations/JupyterReports.sh b/scripts/reports/compilations/JupyterReports.sh
index 11bc3973b..44f7d6d2a 100755
--- a/scripts/reports/compilations/JupyterReports.sh
+++ b/scripts/reports/compilations/JupyterReports.sh
@@ -11,6 +11,10 @@
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+# Overrideable Constants (defaults also defined in sub scripts)
+LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command.
+LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command.
+
 ## Get this "scripts/reports/compilations" directory if not already set.
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -29,9 +33,15 @@ echo "JupyterReports: SCRIPTS_DIR=${SCRIPTS_DIR}"
 JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY:-"${SCRIPTS_DIR}/../jupyter"} # Repository directory containing the Jupyter Notebooks
 echo "JupyterReports: JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY}"
 
-# Run all report scripts
+# Run all jupiter notebooks
 for jupyter_notebook_file in "${JUPYTER_NOTEBOOK_DIRECTORY}"/*.ipynb; do 
-    jupyter_notebook_file=$( basename "${jupyter_notebook_file}")
-    echo "JupyterReports: Executing ${jupyter_notebook_file}..."; 
-    source "${SCRIPTS_DIR}/executeJupyterNotebookReport.sh" --jupyterNotebook "${jupyter_notebook_file}"
+    jupyter_notebook_filename=$(basename -- "${jupyter_notebook_file}")
+    
+    echo "${LOG_GROUP_START}${jupyter_notebook_filename}";
+    echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${jupyter_notebook_filename}...";
+
+    source "${SCRIPTS_DIR}/executeJupyterNotebookReport.sh" --jupyterNotebook "${jupyter_notebook_filename}"
+
+    echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${jupyter_notebook_filename}";
+    echo "${LOG_GROUP_END}";
 done
diff --git a/scripts/reports/compilations/VisualizationReports.sh b/scripts/reports/compilations/VisualizationReports.sh
index 10500ed9f..f8bfc7957 100755
--- a/scripts/reports/compilations/VisualizationReports.sh
+++ b/scripts/reports/compilations/VisualizationReports.sh
@@ -11,6 +11,10 @@
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+# Overrideable Constants (defaults also defined in sub scripts)
+LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command.
+LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command.
+
 ## Get this "scripts/reports/compilations" directory if not already set.
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -21,8 +25,15 @@ echo "VisualizationReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS
 REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCRIPT_DIR}")}
 echo "VisualizationReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}"
 
-# Run all report scripts
-for report_script_file in "${REPORTS_SCRIPT_DIR}"/*Visualization.sh; do 
-    echo "VisualizationReports: Starting ${report_script_file}..."; 
-    source "${report_script_file}"
+# Run all visualization scripts
+for visualization_script_file in "${REPORTS_SCRIPT_DIR}"/*Visualization.sh; do 
+    visualization_script_filename=$(basename -- "${visualization_script_file}")
+    
+    echo "${LOG_GROUP_START}${visualization_script_filename}";
+    echo "VisualizationReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${visualization_script_filename}...";
+
+    source "${visualization_script_file}"
+
+    echo "VisualizationReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${visualization_script_filename}";
+    echo "${LOG_GROUP_END}";
 done