Integrate git log import into analysis preparation

Author: JohT

cypher/GitLog/Find_java_types_not_covered_by_git_data.cypher

@@ -0,0 +1,7 @@
+// Find Java types not covered by git data for git log data import troubleshooting/testing.
+
+MATCH (type:Java&Type&!PrimitiveType&!Void&!JavaType&!ResolvedDuplicateType&!ExternalType)
+WHERE NOT EXISTS { (type)<-[:RESOLVES_TO]-(git_file:Git&File) }
+  AND NOT type.fqn contains '$' // Inner classes are contained in the outer file
+RETURN type.fqn, count(*)
+LIMIT 50

scripts/analysis/analyze.sh

@@ -2,11 +2,11 @@
 
 # Coordinates the end-to-end analysis process, encompassing tool installation, graph generation, and report generation.
 # - Download and setup Neo4j and JQAssistant
-# - Scan and analyze the contents of the artifacts directory to create the graph 
+# - Scan and analyze the contents of the artifacts and source directory to create the graph 
 # - Trigger all requested reports
 
 # Note: Everything is done in the current (=working) directory and one directory above (shared downloads). 
-#       It is recommended to create an empty directory (preferrable "temp") and
+#       It is recommended to create an empty directory (preferable "temp") and
 #       within that another one for the analysis (e.g. "MyCodebaseName-Version")
 #       and change into it prior to starting this script.
 

scripts/downloader/downloadAxonFramework.sh

@@ -15,14 +15,15 @@ SCRIPT_FILE_NAME="$(basename -- "${BASH_SOURCE[0]}")"
 SCRIPT_FILE_NAME_WITHOUT_EXTENSION="${SCRIPT_FILE_NAME%%.*}"
 SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION="${SCRIPT_FILE_NAME_WITHOUT_EXTENSION##download}"
 ANALYSIS_NAME="${SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION}"
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
 
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME=${SCRIPT_FILE_NAME}"
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME_WITHOUT_EXTENSION=${SCRIPT_FILE_NAME_WITHOUT_EXTENSION}"
 echo "download${ANALYSIS_NAME}: ANALYSIS_NAME=${ANALYSIS_NAME}"
 
 # Read the first input argument containing the version(s) of the artifact(s)
 if [ "$#" -ne 1 ]; then
-  echo "Error (download${ANALYSIS_NAME}): Usage: $0 <version>" >&2
+  echo "Error (download${ANALYSIS_NAME}): Usage: $0 <version> (e.g. 4.9.3)" >&2
   exit 1
 fi
 ARTIFACTS_VERSION=$1
@@ -41,12 +42,15 @@ echo "download${ANALYSIS_NAME}: SCRIPTS_DIR=${SCRIPTS_DIR}"
 
 ################################################################
 # Download Artifacts that will be analyzed
-################################################################
 ARTIFACTS_GROUP="org.axonframework"
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-configuration -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-disruptor -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-eventsourcing -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-messaging -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-modelling -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-test -v ${ARTIFACTS_VERSION} || exit 2
+
+# Download the git history (bare clone without working tree) into the "source" folder.
+# This makes it possible to additionally import the git log into the graph 
+git clone --bare https://github.com/AxonFramework/AxonFramework.git --branch "axon-${ARTIFACTS_VERSION}" "${SOURCE_DIRECTORY}/.git"
 ################################################################

scripts/downloader/downloadReactRouter.sh

@@ -11,8 +11,6 @@
 
 # Note: react-router uses pnpm as package manager which needs to be installed
 
-# Requires downloadMavenArtifact.sh
-
 # Fail on any error (errexit = exit on first error, errtrace = error inherited from sub-shell ,pipefail exist on errors within piped commands)
 set -o errexit -o errtrace -o pipefail
 
@@ -21,6 +19,7 @@ SCRIPT_FILE_NAME="$(basename -- "${BASH_SOURCE[0]}")"
 SCRIPT_FILE_NAME_WITHOUT_EXTENSION="${SCRIPT_FILE_NAME%%.*}"
 SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION="${SCRIPT_FILE_NAME_WITHOUT_EXTENSION##download}"
 ANALYSIS_NAME="${SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION}"
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
 
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME=${SCRIPT_FILE_NAME}"
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME_WITHOUT_EXTENSION=${SCRIPT_FILE_NAME_WITHOUT_EXTENSION}"
@@ -40,13 +39,13 @@ mkdir -p ./runtime/logs
 ################################################################
 # Download react-router source files to be analyzed
 ################################################################
-git clone https://github.com/remix-run/react-router.git source
+git clone https://github.com/remix-run/react-router.git "${SOURCE_DIRECTORY}"
 (
-  cd source || exit
+  cd "${SOURCE_DIRECTORY}" || exit
   git checkout "react-router@${PROJECT_VERSION}" || exit
   pnpm install --frozen-lockfile || exit
   npx --yes @jqassistant/ts-lce >./../runtime/logs/jqassostant-typescript-scan.log 2>&1 || exit
 )
 mkdir -p artifacts
-mv -nv "source/.reports/jqa/ts-output.json" "artifacts/ts-react-router-${PROJECT_VERSION}.json"
+mv -nv "${SOURCE_DIRECTORY}/.reports/jqa/ts-output.json" "artifacts/ts-react-router-${PROJECT_VERSION}.json"
 ################################################################

scripts/prepareAnalysis.sh

@@ -2,7 +2,7 @@
 
 # Prepares and validates the graph database before analysis 
 
-# Requires executeQueryFunctions.sh, parseCsvFunctions.sh
+# Requires executeQueryFunctions.sh, parseCsvFunctions.sh, importGitLog.sh, importAggregatedGitLog
 
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
@@ -24,10 +24,10 @@ fi
 CYPHER_DIR=${CYPHER_DIR:-"${SCRIPTS_DIR}/../cypher"} # Repository directory containing the cypher queries
 echo "prepareAnalysis: CYPHER_DIR=${CYPHER_DIR}"
 
-# Define functions to execute a cypher query from within the given file (first and only argument)
+# Define functions (like execute_cypher) to execute a cypher query from within the given file (first and only argument)
 source "${SCRIPTS_DIR}/executeQueryFunctions.sh"
 
-# Define function(s) (e.g. is_csv_column_greater_zero) to parse CSV format strings from Cypher query results.
+# Define functions (like is_csv_column_greater_zero) to parse CSV format strings from Cypher query results.
 source "${SCRIPTS_DIR}/parseCsvFunctions.sh"
 
 # Local Constants
@@ -38,13 +38,18 @@ ARTIFACT_DEPENDENCIES_CYPHER_DIR="$CYPHER_DIR/Artifact_Dependencies"
 TYPES_CYPHER_DIR="$CYPHER_DIR/Types"
 TYPESCRIPT_CYPHER_DIR="$CYPHER_DIR/Typescript_Enrichment"
 
-# Preparation - Data verification: DEPENDS_ON releationships
+# Preparation - Data verification: DEPENDS_ON relationships
 dataVerificationResult=$( execute_cypher "${CYPHER_DIR}/Data_verification_DEPENDS_ON_relationships.cypher" "${@}")
 if ! is_csv_column_greater_zero "${dataVerificationResult}" "sourceNodeCount"; then
     echo "prepareAnalysis: Error: Data verification failed. At least one DEPENDS_ON relationship required. Check if the artifacts directory is empty or if the scan failed."
     exit 1
 fi
 
+# Preparation - Import git log if source or history is available
+# TODO move into separate analysis compilation/part that is selectable
+source "${SCRIPTS_DIR}/importGitLog.sh"
+source "${SCRIPTS_DIR}/importAggregatedGitLog.sh"
+
 # Preparation - Create indices
 execute_cypher "${CYPHER_DIR}/Create_Java_Type_index_for_full_qualified_name.cypher"
 execute_cypher "${CYPHER_DIR}/Create_Typescript_index_for_full_qualified_name.cypher"