Integrate git log import into analysis preparation

Author: JohT

COMMANDS.md

@@ -24,6 +24,7 @@
     - [Setup jQAssistant Java Code Analyzer](#setup-jqassistant-java-code-analyzer)
     - [Download Maven Artifacts to analyze](#download-maven-artifacts-to-analyze)
     - [Reset the database and scan the java artifacts](#reset-the-database-and-scan-the-java-artifacts)
+    - [Import git log](#import-git-log)
 - [Database Queries](#database-queries)
     - [Cypher Shell](#cypher-shell)
     - [HTTP API](#http-api)
@@ -70,7 +71,7 @@ a profile, the newest versions will be used. Profiles are scripts that can be fo
 ### Notes
 
 - Be sure to use Java 17 for Neo4j v5 and Java 11 for Neo4j v4
-- Use your own initial Neo4j password
+- Use your own initial Neo4j password with `export NEO4J_INITIAL_PASSWORD=my_own_password`
 - For more details have a look at the script [analyze.sh](./scripts/analysis/analyze.sh)
 
 ### Examples
@@ -214,6 +215,18 @@ enhance the data further with relationships between artifacts and packages.
 
 Be aware that this script deletes all previous relationships and nodes in the local Neo4j Graph database.
 
+### Import git log
+
+Use [importGitLog.sh](./scripts/importGitLog.sh) to import git log data into the Graph.
+It uses `git log` to extract commits, their authors and the changed filenames into an intermediate CSV file that is then imported into Neo4j with the following schema:
+
+```Cypher
+(Git:Author)-[:AUTHORED]->(Git:Commit)->[:CHANGED]->(Git:File)
+```
+The optional parameter `--repository directory-path-to-a-git-repository` can be used to select a different directory for the repository. By default, the `source` directory within the analysis workspace directory is used. This command only needs the git history to be present so a `git clone --bare` is sufficient. If the `source` directory is also used for the analysis then a full git clone is of course needed (like for Typescript).
+
+👉**Note:** Commit messages containing `[bot]` are filtered out to ignore changes made by bots.
+
 ## Database Queries
 
 ### Cypher Shell

cypher/Create_File_index_for_name.cypher

@@ -0,0 +1,3 @@
+// Create index for the file name
+
+CREATE INDEX INDEX_FILE_NAME IF NOT EXISTS FOR (t:File) ON (t.fileName)

cypher/Imports/Import_git_log_csv_data.cypher

@@ -4,15 +4,15 @@ LOAD CSV WITH HEADERS FROM "file:///gitLog.csv" AS row
 CALL { WITH row
     MERGE (git_author:Git:Author {name: row.author, email: row.email})
     MERGE (git_commit:Git:Commit {
-        hash: row.hash, 
-        message: row.message,
-        timestamp: datetime(row.timestamp),
+        hash:           row.hash, 
+        message:        row.message,
+        timestamp:      datetime(row.timestamp),
         timestamp_unix: toInteger(row.timestamp_unix)
     })
     MERGE (git_file:Git:File {fileName: row.filename})
     MERGE (git_author)-[:AUTHORED]->(git_commit)
     MERGE (git_commit)-[:CHANGED]->(git_file)
 } IN TRANSACTIONS OF 1000 ROWS
-RETURN count(DISTINCT row.author)    AS numberOfAuthors
-       ,count(DISTINCT row.filename) AS numberOfFiles
-       ,count(DISTINCT row.hash)     AS numberOfCommits
+RETURN count(DISTINCT row.author)   AS numberOfAuthors
+      ,count(DISTINCT row.filename) AS numberOfFiles
+      ,count(DISTINCT row.hash)     AS numberOfCommits

cypher/Java_Enrichment/Add_RESOLVES_TO_relationships_to_git_files.cypher

@@ -0,0 +1,12 @@
+// Connect git files to Java files with a RESOLVES_TO relationship if their names match
+
+MATCH (file:File&!Git)
+ WITH file
+     ,replace(file.fileName, '.class', '.java') AS fileName
+MATCH (git_file:File&Git)
+WHERE git_file.fileName ENDS WITH fileName
+MERGE (git_file)-[:RESOLVES_TO]->(file)
+  SET git_file.resolved = true
+RETURN labels(file)[0..4]                AS labels
+       ,count(DISTINCT fileName)         AS numberOfFileNames
+       ,collect(DISTINCT fileName)[0..4] AS examples

cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationships_to_git_files.cypher

@@ -0,0 +1,12 @@
+// Connect git files to Typescript files with a RESOLVES_TO relationship if their names match
+
+MATCH (file:File&!Git)
+ WITH file
+     ,coalesce(file.absoluteFileName, file.globalFqn, file.fileName) AS fileName
+MATCH (git_file:File&Git)
+WHERE fileName ENDS WITH git_file.fileName
+MERGE (git_file)-[:RESOLVES_TO]->(file)
+  SET git_file.resolved = true
+RETURN labels(file)[0..4]               AS labels
+      ,count(DISTINCT fileName)         AS numberOfFileNames
+      ,collect(DISTINCT fileName)[0..4] AS examples

scripts/analysis/analyze.sh

@@ -2,11 +2,11 @@
 
 # Coordinates the end-to-end analysis process, encompassing tool installation, graph generation, and report generation.
 # - Download and setup Neo4j and JQAssistant
-# - Scan and analyze the contents of the artifacts directory to create the graph 
+# - Scan and analyze the contents of the artifacts and source directory to create the graph 
 # - Trigger all requested reports
 
 # Note: Everything is done in the current (=working) directory and one directory above (shared downloads). 
-#       It is recommended to create an empty directory (preferrable "temp") and
+#       It is recommended to create an empty directory (preferable "temp") and
 #       within that another one for the analysis (e.g. "MyCodebaseName-Version")
 #       and change into it prior to starting this script.
 

scripts/downloader/downloadAxonFramework.sh

@@ -15,14 +15,15 @@ SCRIPT_FILE_NAME="$(basename -- "${BASH_SOURCE[0]}")"
 SCRIPT_FILE_NAME_WITHOUT_EXTENSION="${SCRIPT_FILE_NAME%%.*}"
 SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION="${SCRIPT_FILE_NAME_WITHOUT_EXTENSION##download}"
 ANALYSIS_NAME="${SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION}"
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
 
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME=${SCRIPT_FILE_NAME}"
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME_WITHOUT_EXTENSION=${SCRIPT_FILE_NAME_WITHOUT_EXTENSION}"
 echo "download${ANALYSIS_NAME}: ANALYSIS_NAME=${ANALYSIS_NAME}"
 
 # Read the first input argument containing the version(s) of the artifact(s)
 if [ "$#" -ne 1 ]; then
-  echo "Error (download${ANALYSIS_NAME}): Usage: $0 <version>" >&2
+  echo "Error (download${ANALYSIS_NAME}): Usage: $0 <version> (e.g. 4.9.3)" >&2
   exit 1
 fi
 ARTIFACTS_VERSION=$1
@@ -41,12 +42,15 @@ echo "download${ANALYSIS_NAME}: SCRIPTS_DIR=${SCRIPTS_DIR}"
 
 ################################################################
 # Download Artifacts that will be analyzed
-################################################################
 ARTIFACTS_GROUP="org.axonframework"
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-configuration -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-disruptor -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-eventsourcing -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-messaging -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-modelling -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-test -v ${ARTIFACTS_VERSION} || exit 2
+
+# Download the git history (bare clone without working tree) into the "source" folder.
+# This makes it possible to additionally import the git log into the graph 
+git clone --bare https://github.com/AxonFramework/AxonFramework.git --branch "axon-${ARTIFACTS_VERSION}" "${SOURCE_DIRECTORY}/.git"
 ################################################################

scripts/downloader/downloadReactRouter.sh

@@ -11,8 +11,6 @@
 
 # Note: react-router uses pnpm as package manager which needs to be installed
 
-# Requires downloadMavenArtifact.sh
-
 # Fail on any error (errexit = exit on first error, errtrace = error inherited from sub-shell ,pipefail exist on errors within piped commands)
 set -o errexit -o errtrace -o pipefail
 
@@ -21,6 +19,7 @@ SCRIPT_FILE_NAME="$(basename -- "${BASH_SOURCE[0]}")"
 SCRIPT_FILE_NAME_WITHOUT_EXTENSION="${SCRIPT_FILE_NAME%%.*}"
 SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION="${SCRIPT_FILE_NAME_WITHOUT_EXTENSION##download}"
 ANALYSIS_NAME="${SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION}"
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
 
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME=${SCRIPT_FILE_NAME}"
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME_WITHOUT_EXTENSION=${SCRIPT_FILE_NAME_WITHOUT_EXTENSION}"
@@ -40,13 +39,13 @@ mkdir -p ./runtime/logs
 ################################################################
 # Download react-router source files to be analyzed
 ################################################################
-git clone https://github.com/remix-run/react-router.git source
+git clone https://github.com/remix-run/react-router.git "${SOURCE_DIRECTORY}"
 (
-  cd source || exit
+  cd "${SOURCE_DIRECTORY}" || exit
   git checkout "react-router@${PROJECT_VERSION}" || exit
   pnpm install --frozen-lockfile || exit
   npx --yes @jqassistant/ts-lce >./../runtime/logs/jqassostant-typescript-scan.log 2>&1 || exit
 )
 mkdir -p artifacts
-mv -nv "source/.reports/jqa/ts-output.json" "artifacts/ts-react-router-${PROJECT_VERSION}.json"
+mv -nv "${SOURCE_DIRECTORY}/.reports/jqa/ts-output.json" "artifacts/ts-react-router-${PROJECT_VERSION}.json"
 ################################################################

scripts/prepareAnalysis.sh

@@ -2,7 +2,7 @@
 
 # Prepares and validates the graph database before analysis 
 
-# Requires executeQueryFunctions.sh, parseCsvFunctions.sh
+# Requires executeQueryFunctions.sh, parseCsvFunctions.sh, importGitLog.sh
 
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
@@ -24,10 +24,10 @@ fi
 CYPHER_DIR=${CYPHER_DIR:-"${SCRIPTS_DIR}/../cypher"} # Repository directory containing the cypher queries
 echo "prepareAnalysis: CYPHER_DIR=${CYPHER_DIR}"
 
-# Define functions to execute a cypher query from within the given file (first and only argument)
+# Define functions (like execute_cypher) to execute a cypher query from within the given file (first and only argument)
 source "${SCRIPTS_DIR}/executeQueryFunctions.sh"
 
-# Define function(s) (e.g. is_csv_column_greater_zero) to parse CSV format strings from Cypher query results.
+# Define functions (like is_csv_column_greater_zero) to parse CSV format strings from Cypher query results.
 source "${SCRIPTS_DIR}/parseCsvFunctions.sh"
 
 # Local Constants
@@ -37,17 +37,24 @@ EXTERNAL_DEPENDENCIES_CYPHER_DIR="$CYPHER_DIR/External_Dependencies"
 ARTIFACT_DEPENDENCIES_CYPHER_DIR="$CYPHER_DIR/Artifact_Dependencies"
 TYPES_CYPHER_DIR="$CYPHER_DIR/Types"
 TYPESCRIPT_CYPHER_DIR="$CYPHER_DIR/Typescript_Enrichment"
+JAVA_CYPHER_DIR="$CYPHER_DIR/Java_Enrichment"
 
-# Preparation - Data verification: DEPENDS_ON releationships
+# Preparation - Data verification: DEPENDS_ON relationships
 dataVerificationResult=$( execute_cypher "${CYPHER_DIR}/Data_verification_DEPENDS_ON_relationships.cypher" "${@}")
 if ! is_csv_column_greater_zero "${dataVerificationResult}" "sourceNodeCount"; then
     echo "prepareAnalysis: Error: Data verification failed. At least one DEPENDS_ON relationship required. Check if the artifacts directory is empty or if the scan failed."
     exit 1
 fi
 
+# Preparation - Import git log if source or history is available
+source "${SCRIPTS_DIR}/importGitLog.sh"
+execute_cypher "${JAVA_CYPHER_DIR}/Add_RESOLVES_TO_relationships_to_git_files.cypher"
+execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Add_RESOLVES_TO_relationships_to_git_files.cypher"
+
 # Preparation - Create indices
 execute_cypher "${CYPHER_DIR}/Create_Java_Type_index_for_full_qualified_name.cypher"
 execute_cypher "${CYPHER_DIR}/Create_Typescript_index_for_full_qualified_name.cypher"
+execute_cypher "${CYPHER_DIR}/Create_File_index_for_name.cypher"
 
 # Preparation - Enrich Graph for Typescript by adding "module" and "name" properties
 execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Add_name_and_module_properties.cypher"