Use git plugin from kontext-e/jqassistant-plugins

JohT · JohT · commit 417da214ea3c · 2024-08-07T09:36:45.000+02:00
diff --git a/.github/workflows/java-code-analysis.yml b/.github/workflows/java-code-analysis.yml
@@ -131,7 +131,7 @@ jobs:
       env:
         NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }}
         ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: "true"
-        IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "full" # Options: "none", "aggregated", "full"
+        IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or ""
       run: |
         ./../../scripts/analysis/analyze.sh
 
diff --git a/.github/workflows/typescript-code-analysis.yml b/.github/workflows/typescript-code-analysis.yml
@@ -140,7 +140,7 @@ jobs:
       env:
         NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }}
         ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: "true"
-        IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "full" # Options: "none", "aggregated", "full"
+        IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or ""
       run: |
         ./../../scripts/analysis/analyze.sh
 
diff --git a/COMMANDS.md b/COMMANDS.md
@@ -300,7 +300,7 @@ The optional parameter `--source directory-path-to-the-source-folder-containing-
 
 #### Environment Variable
 
-- `IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT` supports the values `none`, `aggregated` and `full` (default). With it, you can switch off git import (`none`), import aggregated data for a smaller memory footprint (`aggregated`) or  import all git commits and the files that where changed with them (`full`=default) .
+- `IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT` supports the values `none`, `aggregated`, `full` and `plugin` (default). With it, you can switch off git import (`none`), import aggregated data for a smaller memory footprint (`aggregated`), import all commits with git log in a simple way (`full`) or let a plugin take care of git data (`plugin`= `""`=default) .
 
 #### Resolving git files to code files
 
diff --git a/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher b/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher
@@ -7,8 +7,11 @@ MATCH (code_file:File&!Git)
 WHERE NOT EXISTS { (code_file)-[:RESOLVES_TO]->(other_file:File&!Git) } // only original nodes, no duplicates
  WITH code_file, replace(code_file.fileName, '.class', '.java') AS codeFileName
 MATCH (git_file:File&Git)
-WHERE git_file.fileName ENDS WITH codeFileName
+ WITH *
+     ,git_file
+     ,coalesce(git_file.fileName, git_file.relativePath) AS gitFileName
+WHERE gitFileName ENDS WITH codeFileName
 MERGE (git_file)-[:RESOLVES_TO]->(code_file)
   SET git_file.resolved = true
 RETURN count(DISTINCT codeFileName)  AS numberOfCodeFiles
-      ,collect(DISTINCT codeFileName + ' <-> ' + git_file.fileName + '\n')[0..4] AS examples
+      ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
diff --git a/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher b/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher
@@ -7,8 +7,11 @@ MATCH (code_file:File&!Git)
 WHERE NOT EXISTS { (code_file)-[:RESOLVES_TO]->(other_file:File&!Git) } // only original nodes, no duplicates
  WITH code_file, code_file.absoluteFileName AS codeFileName
 MATCH (git_file:File&Git)
-WHERE codeFileName      ENDS WITH git_file.fileName
+ WITH *
+     ,git_file
+     ,coalesce(git_file.fileName, git_file.relativePath) AS gitFileName
+WHERE codeFileName      ENDS WITH gitFileName
 MERGE (git_file)-[:RESOLVES_TO]->(code_file)
   SET git_file.resolved = true
 RETURN count(DISTINCT codeFileName)  AS numberOfCodeFiles
-      ,collect(DISTINCT codeFileName + ' <-> ' + git_file.fileName + '\n')[0..4] AS examples
+      ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
diff --git a/cypher/GitLog/Set_number_of_git_log_commits.cypher b/cypher/GitLog/Set_number_of_git_log_commits.cypher
@@ -4,4 +4,5 @@ MATCH (code_file:File&!Git)<-[:RESOLVES_TO]-(git_file:File&Git)
 MATCH (git_file)<-[:CONTAINS_CHANGED]-(git_commit:Git:Commit)
  WITH code_file, count(DISTINCT git_commit.hash) AS numberOfGitCommits
   SET code_file.numberOfGitCommits = numberOfGitCommits
-RETURN count(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName)) AS changedCodeFiles
+RETURN count(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName)) AS changedCodeFiles
+      //,collect(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName))[0..4] AS examples
diff --git a/cypher/GitLog/Set_number_of_git_plugin_commits.cypher b/cypher/GitLog/Set_number_of_git_plugin_commits.cypher
@@ -0,0 +1,8 @@
+// Set numberOfGitCommits property on code File nodes when git commits (detected by the plugin) are present
+
+MATCH (code_file:File&!Git)<-[:RESOLVES_TO]-(git_file:File&Git)
+MATCH (git_file)<-[]-(:Git&Change)<-[:CONTAINS_CHANGE]-(git_commit:Git&Commit)
+ WITH code_file, count(DISTINCT git_commit.sha) AS numberOfGitCommits
+  SET code_file.numberOfGitCommits = numberOfGitCommits
+RETURN count(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName)) AS changedCodeFiles
+      //,collect(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName))[0..4] AS examples
diff --git a/scripts/configuration/template-neo4jv4-jqassistant.yaml b/scripts/configuration/template-neo4jv4-jqassistant.yaml
@@ -33,6 +33,11 @@ jqassistant:
     - group-id: org.jqassistant.plugin
       artifact-id: jqassistant-npm-plugin
       version: 2.0.0
+    - group-id: de.kontext-e.jqassistant.plugin
+      artifact-id: jqassistant.plugin.git
+      version: 2.1.0
+
+
   # The store configuration
   store:
     # URI of the database to connect to. Supported URI schemes are 'file' for embedded databases and 'bolt' for connecting to a running Neo4j instance (3.x+), e.g.
diff --git a/scripts/configuration/template-neo4jv5-jqassistant.yaml b/scripts/configuration/template-neo4jv5-jqassistant.yaml
@@ -33,7 +33,11 @@ jqassistant:
     - group-id: org.jqassistant.plugin
       artifact-id: jqassistant-npm-plugin
       version: 2.0.0
-      
+    - group-id: de.kontext-e.jqassistant.plugin
+      artifact-id: jqassistant.plugin.git
+      version: 2.1.0
+
+
   # The store configuration
   store:
     # URI of the database to connect to. Supported URI schemes are 'file' for embedded databases and 'bolt' for connecting to a running Neo4j instance (3.x+), e.g.
diff --git a/scripts/findPathsToScan.sh b/scripts/findPathsToScan.sh
@@ -40,6 +40,14 @@ if [ -d "${SOURCE_DIRECTORY}" ] ; then
     if [ -n "${npmPackageJsonFiles}" ]; then
         directoriesAndFilesToScan="$(appendNonEmpty "${directoriesAndFilesToScan}")${npmPackageJsonFiles}"
     fi
+
+    # Scan git repositories in the artifacts directory
+    if [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "" ] || [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "plugin" ] ; then
+        gitDirectories="$(find "${SOURCE_DIRECTORY}" -type d -name ".git" -exec echo {} \; | tr '\n' ',' | sed 's/,$/\n/')"
+        if [ -n "${gitDirectories}" ]; then
+            directoriesAndFilesToScan="$(appendNonEmpty "${directoriesAndFilesToScan}")${gitDirectories}"
+        fi
+    fi
 else
     echo "findPathsToScan: Source directory ${SOURCE_DIRECTORY} doesn't exist and will therefore be skipped." >&2
 fi
diff --git a/scripts/importGit.sh b/scripts/importGit.sh
@@ -14,14 +14,14 @@ set -o errexit -o pipefail
 # Overrideable Defaults
 SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
 IMPORT_DIRECTORY=${IMPORT_DIRECTORY:-"import"}
-IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT=${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT:-"full"} # Select how to import git log data. Options: "none", "aggregated", "full". Default="full".
+IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT=${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT:-"plugin"} # Select how to import git log data. Options: "none", "aggregated", "full" and "plugin". Default="plugin".
 
 # Default and initial values for command line options
 source="${SOURCE_DIRECTORY}"
 
 # Read  command line options
 USAGE="importGit: Usage: $0 [--source <directory containing git repositories>(default=source)]"
-while [[ $# -gt 0 ]]; do
+while [ "$#" -gt "0" ]; do
   key="$1"
   case $key in
     --source)
@@ -120,21 +120,28 @@ importAggregatedGitLog() {
   time execute_cypher "${GIT_LOG_CYPHER_DIR}/Import_aggregated_git_log_csv_data.cypher" "${@}"
 }
 
-commonPostGitLogImport() {
+commonPostGitImport() {
   echo "importGit: Creating relationships to nodes with matching file names..."
   execute_cypher "${GIT_LOG_CYPHER_DIR}/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher"
   execute_cypher "${GIT_LOG_CYPHER_DIR}/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher"
 }
 
 postGitLogImport() {
-  commonPostGitLogImport
+  commonPostGitImport
 
   echo "importGit: Add numberOfGitCommits property to nodes with matching file names..."
-  execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_git_commits.cypher"
+  execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_git_log_commits.cypher"
+}
+
+postGitPluginImport() {
+  commonPostGitImport
+
+  echo "importGit: Add numberOfGitCommits property to nodes with matching file names..."
+  execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_git_plugin_commits.cypher"
 }
 
 postAggregatedGitLogImport() {
-  commonPostGitLogImport
+  commonPostGitImport
 
   echo "importGit: Add numberOfGitCommits property to nodes with matching file names..."
   execute_cypher "${GIT_LOG_CYPHER_DIR}/Set_number_of_aggregated_git_commits.cypher"
@@ -146,8 +153,8 @@ mkdir -p "${IMPORT_DIRECTORY}"
 # Internal constants
 NEO4J_FULL_IMPORT_DIRECTORY=$(cd "${IMPORT_DIRECTORY}"; pwd)
 
-# Skip import when IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT = "none"
-if [[ ! ${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT} == "none" ]]; then
+# Skip import if it is switched off ("none") or if it already taken care of by a plugin ("plugin"), which is the default.
+if [ ! "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "none" ] && [ ! "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "plugin" ]; then
 
   existing_data_has_been_deleted=false
 
@@ -163,7 +170,7 @@ if [[ ! ${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT} == "none" ]]; then
     
     create_git_repository_node "${full_repository_path}"
 
-    if [[ ${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT} == "aggregated" ]]; then
+    if [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "aggregated" ]; then
     # Import pre-aggregated git log data (no single commits) when IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT = "aggregated"
         (cd "${repository}" && source "${SCRIPTS_DIR}/createAggregatedGitLogCsv.sh" "${NEO4J_FULL_IMPORT_DIRECTORY}/aggregatedGitLog.csv")
         importAggregatedGitLog "git_repository_absolute_directory_name=${full_repository_path}"
@@ -178,3 +185,8 @@ if [[ ! ${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT} == "none" ]]; then
 else
   echo "importGit: Skipped git import because of IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT=${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}"
 fi
+
+# Even if the data had already been imported by a plugin, the post data enrichment still needs to be done.
+if [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "plugin" ]; then
+  postGitPluginImport
+fi
