Integrate git log import into analysis preparation

JohT · JohT · commit 0fd5cab62776 · 2024-06-09T11:11:33.000+02:00
diff --git a/COMMANDS.md b/COMMANDS.md
@@ -24,6 +24,7 @@
     - [Setup jQAssistant Java Code Analyzer](#setup-jqassistant-java-code-analyzer)
     - [Download Maven Artifacts to analyze](#download-maven-artifacts-to-analyze)
     - [Reset the database and scan the java artifacts](#reset-the-database-and-scan-the-java-artifacts)
+    - [Import git log](#import-git-log)
 - [Database Queries](#database-queries)
     - [Cypher Shell](#cypher-shell)
     - [HTTP API](#http-api)
@@ -70,7 +71,7 @@ a profile, the newest versions will be used. Profiles are scripts that can be fo
 ### Notes
 
 - Be sure to use Java 17 for Neo4j v5 and Java 11 for Neo4j v4
-- Use your own initial Neo4j password
+- Use your own initial Neo4j password with `export NEO4J_INITIAL_PASSWORD=my_own_password`
 - For more details have a look at the script [analyze.sh](./scripts/analysis/analyze.sh)
 
 ### Examples
@@ -214,6 +215,18 @@ enhance the data further with relationships between artifacts and packages.
 
 Be aware that this script deletes all previous relationships and nodes in the local Neo4j Graph database.
 
+### Import git log
+
+Use [importGitLog.sh](./scripts/importGitLog.sh) to import git log data into the Graph.
+It uses `git log` to extract commits, their authors and the changed filenames into an intermediate CSV file that is then imported into Neo4j with the following schema:
+
+```Cypher
+(Git:Author)-[:AUTHORED]->(Git:Commit)->[:CHANGED]->(Git:File)
+```
+The optional parameter `--repository directory-path-to-a-git-repository` can be used to select a different directory for the repository. By default, the `source` directory within the analysis workspace directory is used. This command only needs the git history to be present so a `git clone --bare` is sufficient. If the `source` directory is also used for the analysis then a full git clone is of course needed (like for Typescript).
+
+👉**Note:** Commit messages containing `[bot]` are filtered out to ignore changes made by bots.
+
 ## Database Queries
 
 ### Cypher Shell
diff --git a/cypher/Create_File_index_for_name.cypher b/cypher/Create_File_index_for_name.cypher
@@ -0,0 +1,3 @@
+// Create index for the file name
+
+CREATE INDEX INDEX_FILE_NAME IF NOT EXISTS FOR (t:File) ON (t.fileName)
diff --git a/cypher/Imports/Import_git_log_csv_data.cypher b/cypher/Imports/Import_git_log_csv_data.cypher
@@ -4,15 +4,15 @@ LOAD CSV WITH HEADERS FROM "file:///gitLog.csv" AS row
 CALL { WITH row
     MERGE (git_author:Git:Author {name: row.author, email: row.email})
     MERGE (git_commit:Git:Commit {
-        hash: row.hash, 
-        message: row.message,
-        timestamp: datetime(row.timestamp),
+        hash:           row.hash, 
+        message:        row.message,
+        timestamp:      datetime(row.timestamp),
         timestamp_unix: toInteger(row.timestamp_unix)
     })
     MERGE (git_file:Git:File {fileName: row.filename})
     MERGE (git_author)-[:AUTHORED]->(git_commit)
     MERGE (git_commit)-[:CHANGED]->(git_file)
 } IN TRANSACTIONS OF 1000 ROWS
-RETURN count(DISTINCT row.author)    AS numberOfAuthors
-       ,count(DISTINCT row.filename) AS numberOfFiles
-       ,count(DISTINCT row.hash)     AS numberOfCommits
+RETURN count(DISTINCT row.author)   AS numberOfAuthors
+      ,count(DISTINCT row.filename) AS numberOfFiles
+      ,count(DISTINCT row.hash)     AS numberOfCommits
diff --git a/cypher/Java_Enrichment/Add_RESOLVES_TO_relationships_to_git_files.cypher b/cypher/Java_Enrichment/Add_RESOLVES_TO_relationships_to_git_files.cypher
@@ -0,0 +1,12 @@
+// Connect git files to Java files with a RESOLVES_TO relationship if their names match
+
+MATCH (file:File&!Git)
+ WITH file
+     ,replace(file.fileName, '.class', '.java') AS fileName
+MATCH (git_file:File&Git)
+WHERE git_file.fileName ENDS WITH fileName
+MERGE (git_file)-[:RESOLVES_TO]->(file)
+  SET git_file.resolved = true
+RETURN labels(file)[0..4]                AS labels
+       ,count(DISTINCT fileName)         AS numberOfFileNames
+       ,collect(DISTINCT fileName)[0..4] AS examples
diff --git a/cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationships_to_git_files.cypher b/cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationships_to_git_files.cypher
@@ -0,0 +1,12 @@
+// Connect git files to Typescript files with a RESOLVES_TO relationship if their names match
+
+MATCH (file:File&!Git)
+ WITH file
+     ,coalesce(file.absoluteFileName, file.globalFqn, file.fileName) AS fileName
+MATCH (git_file:File&Git)
+WHERE fileName ENDS WITH git_file.fileName
+MERGE (git_file)-[:RESOLVES_TO]->(file)
+  SET git_file.resolved = true
+RETURN labels(file)[0..4]               AS labels
+      ,count(DISTINCT fileName)         AS numberOfFileNames
+      ,collect(DISTINCT fileName)[0..4] AS examples
diff --git a/scripts/analysis/analyze.sh b/scripts/analysis/analyze.sh
@@ -2,11 +2,11 @@
 
 # Coordinates the end-to-end analysis process, encompassing tool installation, graph generation, and report generation.
 # - Download and setup Neo4j and JQAssistant
-# - Scan and analyze the contents of the artifacts directory to create the graph 
+# - Scan and analyze the contents of the artifacts and source directory to create the graph 
 # - Trigger all requested reports
 
 # Note: Everything is done in the current (=working) directory and one directory above (shared downloads). 
-#       It is recommended to create an empty directory (preferrable "temp") and
+#       It is recommended to create an empty directory (preferable "temp") and
 #       within that another one for the analysis (e.g. "MyCodebaseName-Version")
 #       and change into it prior to starting this script.
 
diff --git a/scripts/downloader/downloadAxonFramework.sh b/scripts/downloader/downloadAxonFramework.sh
@@ -15,14 +15,15 @@ SCRIPT_FILE_NAME="$(basename -- "${BASH_SOURCE[0]}")"
 SCRIPT_FILE_NAME_WITHOUT_EXTENSION="${SCRIPT_FILE_NAME%%.*}"
 SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION="${SCRIPT_FILE_NAME_WITHOUT_EXTENSION##download}"
 ANALYSIS_NAME="${SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION}"
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
 
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME=${SCRIPT_FILE_NAME}"
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME_WITHOUT_EXTENSION=${SCRIPT_FILE_NAME_WITHOUT_EXTENSION}"
 echo "download${ANALYSIS_NAME}: ANALYSIS_NAME=${ANALYSIS_NAME}"
 
 # Read the first input argument containing the version(s) of the artifact(s)
 if [ "$#" -ne 1 ]; then
-  echo "Error (download${ANALYSIS_NAME}): Usage: $0 <version>" >&2
+  echo "Error (download${ANALYSIS_NAME}): Usage: $0 <version> (e.g. 4.9.3)" >&2
   exit 1
 fi
 ARTIFACTS_VERSION=$1
@@ -41,12 +42,15 @@ echo "download${ANALYSIS_NAME}: SCRIPTS_DIR=${SCRIPTS_DIR}"
 
 ################################################################
 # Download Artifacts that will be analyzed
-################################################################
 ARTIFACTS_GROUP="org.axonframework"
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-configuration -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-disruptor -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-eventsourcing -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-messaging -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-modelling -v ${ARTIFACTS_VERSION} || exit 2
 source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g ${ARTIFACTS_GROUP} -a axon-test -v ${ARTIFACTS_VERSION} || exit 2
+
+# Download the git history (bare clone without working tree) into the "source" folder.
+# This makes it possible to additionally import the git log into the graph 
+git clone --bare https://github.com/AxonFramework/AxonFramework.git --branch "axon-${ARTIFACTS_VERSION}" "${SOURCE_DIRECTORY}/.git"
 ################################################################
diff --git a/scripts/downloader/downloadReactRouter.sh b/scripts/downloader/downloadReactRouter.sh
@@ -11,8 +11,6 @@
 
 # Note: react-router uses pnpm as package manager which needs to be installed
 
-# Requires downloadMavenArtifact.sh
-
 # Fail on any error (errexit = exit on first error, errtrace = error inherited from sub-shell ,pipefail exist on errors within piped commands)
 set -o errexit -o errtrace -o pipefail
 
@@ -21,6 +19,7 @@ SCRIPT_FILE_NAME="$(basename -- "${BASH_SOURCE[0]}")"
 SCRIPT_FILE_NAME_WITHOUT_EXTENSION="${SCRIPT_FILE_NAME%%.*}"
 SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION="${SCRIPT_FILE_NAME_WITHOUT_EXTENSION##download}"
 ANALYSIS_NAME="${SCRIPT_FILE_NAME_WITHOUT_PREFIX_AND_EXTENSION}"
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
 
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME=${SCRIPT_FILE_NAME}"
 echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME_WITHOUT_EXTENSION=${SCRIPT_FILE_NAME_WITHOUT_EXTENSION}"
@@ -40,13 +39,13 @@ mkdir -p ./runtime/logs
 ################################################################
 # Download react-router source files to be analyzed
 ################################################################
-git clone https://github.com/remix-run/react-router.git source
+git clone https://github.com/remix-run/react-router.git "${SOURCE_DIRECTORY}"
 (
-  cd source || exit
+  cd "${SOURCE_DIRECTORY}" || exit
   git checkout "react-router@${PROJECT_VERSION}" || exit
   pnpm install --frozen-lockfile || exit
   npx --yes @jqassistant/ts-lce >./../runtime/logs/jqassostant-typescript-scan.log 2>&1 || exit
 )
 mkdir -p artifacts
-mv -nv "source/.reports/jqa/ts-output.json" "artifacts/ts-react-router-${PROJECT_VERSION}.json"
+mv -nv "${SOURCE_DIRECTORY}/.reports/jqa/ts-output.json" "artifacts/ts-react-router-${PROJECT_VERSION}.json"
 ################################################################
diff --git a/scripts/importGitLog.sh b/scripts/importGitLog.sh
@@ -83,13 +83,36 @@ echo "importGitLog: Creating ${OUTPUT_CSV_FILENAME} from git log..."
   
   # Prints the git log in CSV format including the changed files.
   # Includes quoted strings, double quote escaping and supports commas in strings.
-  # - --pretty=format starts with a space that is needed to detect the start of a line.
-  #   gsub(/^ /, "", a[1]); removes that space then afterwards   
-  # - 3 commas (,,,) should be very unlikely to appear in names, email addresses and commit messages so they are used as an intermediate separator (see split)
-  # - gsub(/"/, "\"\"", a[6]) escapes double quotes with two of them (CSV standard)
   git log --no-merges --pretty=format:' %h,,,%an,,,%ae,,,%aI,,,%ct,,,%s' --name-only | \
   awk 'BEGIN { COMMA=",";QUOTE="\"" } /^ / { split($0, a, ",,,"); gsub(/^ /, "", a[1]); gsub(/"/, "\"\"", a[2]); gsub(/"/, "\"\"", a[3]); gsub(/"/, "\"\"", a[6]); gsub(/\\/, " ", a[6]); commit=a[1] COMMA QUOTE a[2] QUOTE COMMA QUOTE a[3] QUOTE COMMA a[4] COMMA a[5] COMMA QUOTE a[6] QUOTE } NF && !/^\ / { print commit ",\""$0"\"" }' | \
   grep -v -F '[bot]' >> "${OUTPUT_CSV_FILENAME}"
+  # Explanation:
+  #
+  # - --no-merges: Excludes merge commits from the log.
+  # - %h: Abbreviated commit hash
+  # - %an: Author name
+  # - %ae: Author email
+  # - %aI: Author date, ISO 8601 format
+  # - %ct: Commit date, Unix timestamp
+  # - %s: Subject of the commit
+  # - --name-only: Lists the files affected by each commit.
+  # - --pretty=format starts with a space that is needed to detect the start of a line.
+  # - The chosen delimiters ,,, are used to separate these fields to make parsing easier.
+  #   It is very unlikely that they appear in the contents and will be used as an intermediate step before escaping.
+  #
+  # - BEGIN { COMMA=","; QUOTE="\"" }: Initializes the variables COMMA and QUOTE to hold a comma and a double-quote character respectively.
+  # - /^ / { ... }: Processes lines that start with a space (indicating a file name in git log --name-only output).
+  # - gsub(/^ /, "", a[1]): Removes leading spaces from the first field (commit hash) that was used to indicate a new commit.
+  # - gsub(/"/, "\"\"", a[6]) escapes double quotes with two double quotes (CSV standard).
+  #   a[6] is the commit message column. Double quote escaping is done for every string column
+  # - gsub(/\\/, " ", a[6]): Replaces backslashes in the commit message with spaces.
+  #   Otherwise, \" would lead to an error since it would be seen as an non escaped double quote.
+  # - commit=...: Constructs the commit information in CSV format, including the quoted author name, author email, and commit message except for the file name.
+  # - NF && !/^\ / { print commit ",\""$0"\"" }: For non-empty lines that do not start with a space (indicating commit information), 
+  #   it prints the commit information followed by the file name(s), enclosed in quotes.
+  #
+  # - grep -v -F '[bot]': Filters out commits where the commit message includes [bot]
+  #   Used to identify commits made by automated systems or bots.
 )
 
 csv_file_size=$(wc -c "${OUTPUT_CSV_FILENAME}" | awk '{print $1}')
diff --git a/scripts/prepareAnalysis.sh b/scripts/prepareAnalysis.sh
@@ -2,7 +2,7 @@
 
 # Prepares and validates the graph database before analysis 
 
-# Requires executeQueryFunctions.sh, parseCsvFunctions.sh
+# Requires executeQueryFunctions.sh, parseCsvFunctions.sh, importGitLog.sh
 
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
@@ -24,10 +24,10 @@ fi
 CYPHER_DIR=${CYPHER_DIR:-"${SCRIPTS_DIR}/../cypher"} # Repository directory containing the cypher queries
 echo "prepareAnalysis: CYPHER_DIR=${CYPHER_DIR}"
 
-# Define functions to execute a cypher query from within the given file (first and only argument)
+# Define functions (like execute_cypher) to execute a cypher query from within the given file (first and only argument)
 source "${SCRIPTS_DIR}/executeQueryFunctions.sh"
 
-# Define function(s) (e.g. is_csv_column_greater_zero) to parse CSV format strings from Cypher query results.
+# Define functions (like is_csv_column_greater_zero) to parse CSV format strings from Cypher query results.
 source "${SCRIPTS_DIR}/parseCsvFunctions.sh"
 
 # Local Constants
@@ -37,17 +37,24 @@ EXTERNAL_DEPENDENCIES_CYPHER_DIR="$CYPHER_DIR/External_Dependencies"
 ARTIFACT_DEPENDENCIES_CYPHER_DIR="$CYPHER_DIR/Artifact_Dependencies"
 TYPES_CYPHER_DIR="$CYPHER_DIR/Types"
 TYPESCRIPT_CYPHER_DIR="$CYPHER_DIR/Typescript_Enrichment"
+JAVA_CYPHER_DIR="$CYPHER_DIR/Java_Enrichment"
 
-# Preparation - Data verification: DEPENDS_ON releationships
+# Preparation - Data verification: DEPENDS_ON relationships
 dataVerificationResult=$( execute_cypher "${CYPHER_DIR}/Data_verification_DEPENDS_ON_relationships.cypher" "${@}")
 if ! is_csv_column_greater_zero "${dataVerificationResult}" "sourceNodeCount"; then
     echo "prepareAnalysis: Error: Data verification failed. At least one DEPENDS_ON relationship required. Check if the artifacts directory is empty or if the scan failed."
     exit 1
 fi
 
+# Preparation - Import git log if source or history is available
+source "${SCRIPTS_DIR}/importGitLog.sh"
+execute_cypher "${JAVA_CYPHER_DIR}/Add_RESOLVES_TO_relationships_to_git_files.cypher"
+execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Add_RESOLVES_TO_relationships_to_git_files.cypher"
+
 # Preparation - Create indices
 execute_cypher "${CYPHER_DIR}/Create_Java_Type_index_for_full_qualified_name.cypher"
 execute_cypher "${CYPHER_DIR}/Create_Typescript_index_for_full_qualified_name.cypher"
+execute_cypher "${CYPHER_DIR}/Create_File_index_for_name.cypher"
 
 # Preparation - Create DEPENDS_ON for every DEPENDS_ON_* relationship
 # Workaround for https://github.com/jQAssistant/jqa-java-plugin/issues/44

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+// Create index for the file name`
	`2`	`+`
	`3`	`+CREATE INDEX INDEX_FILE_NAME IF NOT EXISTS FOR (t:File) ON (t.fileName)`