Merge pull request #195 from JohT/feature/use-kontext-e-jqassistant-git-plugin

JohT · web-flow · commit 0669ff779783 · 2024-08-08T08:47:32.000+02:00
Support scanning files outside "artifacts" and switch to plugin for git data
diff --git a/.github/workflows/java-code-analysis.yml b/.github/workflows/java-code-analysis.yml
@@ -131,7 +131,7 @@ jobs:
       env:
         NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }}
         ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: "true"
-        IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "full" # Options: "none", "aggregated", "full"
+        IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or ""
       run: |
         ./../../scripts/analysis/analyze.sh
 
diff --git a/.github/workflows/typescript-code-analysis.yml b/.github/workflows/typescript-code-analysis.yml
@@ -140,7 +140,7 @@ jobs:
       env:
         NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }}
         ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: "true"
-        IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "full" # Options: "none", "aggregated", "full"
+        IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or ""
       run: |
         ./../../scripts/analysis/analyze.sh
 
diff --git a/COMMANDS.md b/COMMANDS.md
@@ -300,7 +300,7 @@ The optional parameter `--source directory-path-to-the-source-folder-containing-
 
 #### Environment Variable
 
-- `IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT` supports the values `none`, `aggregated` and `full` (default). With it, you can switch off git import (`none`), import aggregated data for a smaller memory footprint (`aggregated`) or  import all git commits and the files that where changed with them (`full`=default) .
+- `IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT` supports the values `none`, `aggregated`, `full` and `plugin` (default). With it, you can switch off git import (`none`), import aggregated data for a smaller memory footprint (`aggregated`), import all commits with git log in a simple way (`full`) or let a plugin take care of git data (`plugin`= `""`=default) .
 
 #### Resolving git files to code files
 
diff --git a/README.md b/README.md
@@ -91,9 +91,7 @@ This could be as simple as running the following command in your Typescript proj
   npx --yes @jqassistant/ts-lce
   ```
 
-- It is recommended to put the cloned source code repository into a directory called `source` within the analysis workspace so that it will also be picked up to import git log data.
-
-- Copy the resulting json file (e.g. `.reports/jqa/ts-output.json`) into the `artifacts/typescript` directory for your analysis work directory. Create the directory, if it doesn't exists. Custom subdirectories within `artifacts/typescript` are also supported.
+- The cloned repository or source project needs to be copied into the directory called `source` within the analysis workspace, so that it will also be picked up during scan by [resetAndScan.sh](./scripts/resetAndScan.sh) and optional [importGit.sh](./scripts/importGit.sh).
 
 ## :rocket: Getting Started
 
diff --git a/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher b/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher
@@ -7,8 +7,11 @@ MATCH (code_file:File&!Git)
 WHERE NOT EXISTS { (code_file)-[:RESOLVES_TO]->(other_file:File&!Git) } // only original nodes, no duplicates
  WITH code_file, replace(code_file.fileName, '.class', '.java') AS codeFileName
 MATCH (git_file:File&Git)
-WHERE git_file.fileName ENDS WITH codeFileName
+ WITH *
+     ,git_file
+     ,coalesce(git_file.fileName, git_file.relativePath) AS gitFileName
+WHERE gitFileName ENDS WITH codeFileName
 MERGE (git_file)-[:RESOLVES_TO]->(code_file)
   SET git_file.resolved = true
 RETURN count(DISTINCT codeFileName)  AS numberOfCodeFiles
-      ,collect(DISTINCT codeFileName + ' <-> ' + git_file.fileName + '\n')[0..4] AS examples
+      ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
diff --git a/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher b/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher
@@ -7,8 +7,11 @@ MATCH (code_file:File&!Git)
 WHERE NOT EXISTS { (code_file)-[:RESOLVES_TO]->(other_file:File&!Git) } // only original nodes, no duplicates
  WITH code_file, code_file.absoluteFileName AS codeFileName
 MATCH (git_file:File&Git)
-WHERE codeFileName      ENDS WITH git_file.fileName
+ WITH *
+     ,git_file
+     ,coalesce(git_file.fileName, git_file.relativePath) AS gitFileName
+WHERE codeFileName      ENDS WITH gitFileName
 MERGE (git_file)-[:RESOLVES_TO]->(code_file)
   SET git_file.resolved = true
 RETURN count(DISTINCT codeFileName)  AS numberOfCodeFiles
-      ,collect(DISTINCT codeFileName + ' <-> ' + git_file.fileName + '\n')[0..4] AS examples
+      ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
diff --git a/cypher/GitLog/Import_git_log_csv_data.cypher b/cypher/GitLog/Import_git_log_csv_data.cypher
@@ -5,6 +5,7 @@ CALL { WITH row
     MATCH (git_repository:Git:Repository{absoluteFileName: $git_repository_absolute_directory_name})
     MERGE (git_author:Git:Log:Author {name: row.author, email: row.email})
     MERGE (git_commit:Git:Log:Commit {
+        sha:            row.hash, 
         hash:           row.hash, 
         parent:         coalesce(row.parent, ''), 
         message:        row.message,
diff --git a/cypher/GitLog/Index_commit_sha.cypher b/cypher/GitLog/Index_commit_sha.cypher
@@ -0,0 +1,3 @@
+// Create index for git commit sha
+
+CREATE INDEX INDEX_COMMIT_HASH IF NOT EXISTS FOR (commit:Commit) ON (commit.sha)
diff --git a/cypher/GitLog/Index_file_relative_path.cypher b/cypher/GitLog/Index_file_relative_path.cypher
@@ -0,0 +1,3 @@
+// Create index for the relative file path
+
+CREATE INDEX INDEX_FILE_NAME IF NOT EXISTS FOR (file:File) ON (file.relativePath)
diff --git a/cypher/GitLog/Set_number_of_git_log_commits.cypher b/cypher/GitLog/Set_number_of_git_log_commits.cypher
@@ -4,4 +4,5 @@ MATCH (code_file:File&!Git)<-[:RESOLVES_TO]-(git_file:File&Git)
 MATCH (git_file)<-[:CONTAINS_CHANGED]-(git_commit:Git:Commit)
  WITH code_file, count(DISTINCT git_commit.hash) AS numberOfGitCommits
   SET code_file.numberOfGitCommits = numberOfGitCommits
-RETURN count(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName)) AS changedCodeFiles
+RETURN count(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName)) AS changedCodeFiles
+      //,collect(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName))[0..4] AS examples
diff --git a/cypher/GitLog/Set_number_of_git_plugin_commits.cypher b/cypher/GitLog/Set_number_of_git_plugin_commits.cypher
@@ -0,0 +1,8 @@
+// Set numberOfGitCommits property on code File nodes when git commits (detected by the plugin) are present
+
+MATCH (code_file:File&!Git)<-[:RESOLVES_TO]-(git_file:File&Git)
+MATCH (git_file)<-[]-(:Git&Change)<-[:CONTAINS_CHANGE]-(git_commit:Git&Commit)
+ WITH code_file, count(DISTINCT git_commit.sha) AS numberOfGitCommits
+  SET code_file.numberOfGitCommits = numberOfGitCommits
+RETURN count(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName)) AS changedCodeFiles
+      //,collect(DISTINCT coalesce(code_file.absoluteFileName, code_file.fileName))[0..4] AS examples
diff --git a/renovate.json b/renovate.json
@@ -119,8 +119,6 @@
     },
     {
       "fileMatch": [
-        "^scripts\/profiles\/Neo4jv5\\.sh$",
-        "^scripts\/profiles\/Default\\.sh$",
         "^scripts\/[^\/]*\\.sh$"
       ],
       "matchStrings": [
diff --git a/scripts/analysis/analyze.sh b/scripts/analysis/analyze.sh
@@ -38,6 +38,7 @@ REPORTS_SCRIPTS_DIRECTORY=${REPORTS_SCRIPTS_DIRECTORY:-"reports"} # Working dire
 REPORT_COMPILATIONS_SCRIPTS_DIRECTORY=${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY:-"compilations"} # Repository directory that contains scripts that execute selected report generation scripts
 SETTINGS_PROFILE_SCRIPTS_DIRECTORY=${SETTINGS_PROFILE_SCRIPTS_DIRECTORY:-"profiles"} # Repository directory that contains scripts containing settings
 ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"} # Working directory containing the artifacts to be analyzed
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"}
 
 # Function to display script usage
 usage() {
@@ -86,9 +87,9 @@ if ! [[ ${settingsProfile} =~ ^[[:alnum:]]+$ ]]; then
   exit 1
 fi
 
-# Check if Neo4j is installed
-if [ ! -d "${ARTIFACTS_DIRECTORY}" ] ; then
-    echo "analyze: The ${ARTIFACTS_DIRECTORY} directory doesn't exist. Please download artifacts first."
+# Check if there is something to scan and analyze
+if [ ! -d "${ARTIFACTS_DIRECTORY}" ] && [ ! -d "${SOURCE_DIRECTORY}" ] ; then
+    echo "analyze: Neither ${ARTIFACTS_DIRECTORY} nor the ${SOURCE_DIRECTORY} directory exist. Please download artifacts/sources first."
     exit 1
 fi
 
diff --git a/scripts/configuration/template-neo4jv4-jqassistant.yaml b/scripts/configuration/template-neo4jv4-jqassistant.yaml
@@ -33,6 +33,11 @@ jqassistant:
     - group-id: org.jqassistant.plugin
       artifact-id: jqassistant-npm-plugin
       version: 2.0.0
+    - group-id: de.kontext-e.jqassistant.plugin
+      artifact-id: jqassistant.plugin.git
+      version: 2.1.0
+
+
   # The store configuration
   store:
     # URI of the database to connect to. Supported URI schemes are 'file' for embedded databases and 'bolt' for connecting to a running Neo4j instance (3.x+), e.g.
@@ -94,7 +99,6 @@ jqassistant:
       #
       # -Djqassistant.store.properties
       properties:
-        json.file.exclude: "*/typescript/*.json"
 
   # The Scanner configuration
   scan:
@@ -127,6 +131,7 @@ jqassistant:
     # -Djqassistant.scan.properties
     properties:
       # plugin.property.key: value
+      json.file.exclude: "*/.reports/jqa/ts-output.json"
 
   # The analyze configuration
   analyze:
diff --git a/scripts/configuration/template-neo4jv5-jqassistant.yaml b/scripts/configuration/template-neo4jv5-jqassistant.yaml
@@ -33,7 +33,11 @@ jqassistant:
     - group-id: org.jqassistant.plugin
       artifact-id: jqassistant-npm-plugin
       version: 2.0.0
-      
+    - group-id: de.kontext-e.jqassistant.plugin
+      artifact-id: jqassistant.plugin.git
+      version: 2.1.0
+
+
   # The store configuration
   store:
     # URI of the database to connect to. Supported URI schemes are 'file' for embedded databases and 'bolt' for connecting to a running Neo4j instance (3.x+), e.g.
@@ -127,7 +131,7 @@ jqassistant:
     # -Djqassistant.scan.properties
     properties:
       # plugin.property.key: value
-      json.file.exclude: "*/typescript/*.json"
+      json.file.exclude: "*/.reports/jqa/ts-output.json"
 
   # The analyze configuration
   analyze:
diff --git a/scripts/copyPackageJsonFiles.sh b/scripts/copyPackageJsonFiles.sh
@@ -31,7 +31,7 @@ fi
         # echo "copyPackageJsonFiles: Copying ${file} to ${targetDirectory}"
 
         mkdir -p "${targetDirectory}"
-        cp "${file}" "${targetDirectory}"
+        cp -rf "${file}" "${targetDirectory}"
 
         # Workaround until the following issue is resolved:
         # https://github.com/jqassistant-plugin/jqassistant-npm-plugin/issues/5
diff --git a/scripts/detectChangedArtifacts.sh b/scripts/detectChangedArtifacts.sh
@@ -1,32 +1,45 @@
 #!/usr/bin/env bash
 
-# Detect changed files in the artifacts directory with a text file containing the last hash code of the contents.
+# Detect changed files in the artifacts directory or in a given list of paths 
+# using a text file containing the last hash code of the contents.
 # The hash value is generated based on all files (their names and properties) within the artifacts directory.
 # A change is detected when the current hash and the stored differ.
 #
 # Command line options:
 # --readonly Detect changes without creating or updating the change detection file (stateless).
 #            A second call without this option will be needed for the change detection to work.
 #            This is helpful to decide if an operation should be done based on changes while waiting for its success to finally save the change state.
+# --paths Comma-separated list of file- and directory-names that are used for calculating the hash based on their name and size.
 
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+# Overrideable defaults
+ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"}
+ARTIFACTS_CHANGE_DETECTION_HASH_FILE=${ARTIFACTS_CHANGE_DETECTION_HASH_FILE:-"artifactsChangeDetectionHash.txt"} # Name of the file that contains the hash code of the file list for change detection
+ARTIFACTS_CHANGE_DETECTION_HASH_FILE_PATH="./${ARTIFACTS_DIRECTORY}/$ARTIFACTS_CHANGE_DETECTION_HASH_FILE"
+
 # Function to display script usage
 usage() {
-  echo "Usage: $0 [--readonly]"
+  echo "Usage: $0 [--readonly] [--paths <comma separated list of file and directory names>]"
   exit 1
 }
 
 # Default values
 readonlyMode=false
+paths="./${ARTIFACTS_DIRECTORY}"
 
 # Parse command line arguments
 while [[ $# -gt 0 ]]; do
   key="$1"
-  case $key in
+  value="${2}"
+
+  case ${key} in
     --readonly)
       readonlyMode=true
+      ;;
+    --paths)
+      paths="${value}"
       shift
       ;;
     *)
@@ -41,42 +54,84 @@ if ${readonlyMode}; then
   echo "detectChangedArtifacts: Readonly mode activated. Change detection file won't be created." >&2
 fi
 
-ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"}
-ARTIFACTS_CHANGE_DETECTION_HASH_FILE=${ARTIFACTS_CHANGE_DETECTION_HASH_FILE:-"artifactsChangeDetectionHash.txt"} # Name of the file that contains the hash code of the file list for change detection
-ARTIFACTS_CHANGE_DETECTION_HASH_FILE_PATH="./${ARTIFACTS_DIRECTORY}/$ARTIFACTS_CHANGE_DETECTION_HASH_FILE"
-
 # Check if the artifacts directory exists
-if [ ! -d "./${ARTIFACTS_DIRECTORY}" ] ; then
-    echo 0 # The artifact directory doesn't exist. There is nothing to compare. Therefore assume that there are no changes.
+if [ -z "${paths}" ] ; then
+    echo 0 # 0=No change detected. The path list is empty. There is nothing to compare. Therefore assume that there are no changes.
     exit 0
 fi
 
+# Function to get file size
+get_file_size() {
+    if [ -f "$1" ]; then
+        wc -c < "$1" | tr -d ' '
+    else
+        echo 0
+    fi
+}
+
+# Function to process a single path
+unwind_directories() {
+    if [ -d "$1" ]; then
+        # If it's a directory, list all files inside 
+        # except for "node_modules", "target", "temp" and the change detection file itself
+        find "$1" \
+          -type d -name "node_modules" -prune -o \
+          -type d -name "target" -prune -o \
+          -type d -name "temp" -prune -o \
+          -not -name "${ARTIFACTS_CHANGE_DETECTION_HASH_FILE}" \
+          -type f \
+        | sort 
+    elif [ -f "$1" ]; then
+        # If it's a file, just echo the file path
+        echo "$1"
+    fi
+}
+
+# Function that takes a comma-separated list of file- and directory-names, 
+# finds all files in the directories 
+# and calculates the md5 hash for every of these .
+get_md5_checksum_of_all_file_names_and_sizes() {
+  local paths=${1}
+  local files_and_sizes=""
+
+  for path in ${paths//,/ }; do
+      files=$(unwind_directories "${path}")
+      for file in ${files}; do
+          size=$(get_file_size "${file}")
+          files_and_sizes="${files_and_sizes}${file}${size}"
+      done
+  done
+
+  echo "${files_and_sizes}" | openssl md5 | awk '{print $2}'
+}
+
 # Use find to list all files in the directory with their properties,
 # sort the output, and pipe it to md5 to create a hash
 # Use openssl md5 that is at least available on Mac and Linux. 
 # See: https://github.com/TRON-US/go-btfs/issues/90#issuecomment-517409369
-CURRENT_ARTIFACTS_HASH="$( find "./$ARTIFACTS_DIRECTORY" -type f -not -name "${ARTIFACTS_CHANGE_DETECTION_HASH_FILE}" -exec openssl md5 -binary {} + | openssl md5 | awk '{print $2}' )"
+CURRENT_ARTIFACTS_HASH=$(get_md5_checksum_of_all_file_names_and_sizes "${paths}")
 
 # Assume that the files where changed if the file containing the hash of the file list does not exist yet.
 if [ ! -f "${ARTIFACTS_CHANGE_DETECTION_HASH_FILE_PATH}" ] ; then
-    if ! ${readonlyMode}; then
+    if [ "${readonlyMode}" = false ] ; then
         # Create the file containing the hash of the files list to a new file for the next call
+        mkdir -p "${ARTIFACTS_DIRECTORY}"
         echo "${CURRENT_ARTIFACTS_HASH}" > "${ARTIFACTS_CHANGE_DETECTION_HASH_FILE_PATH}"
         echo "detectChangedArtifacts: Change detection file created" >&2
     fi
-    echo 1
+    echo 1 # 1=Change detected and change detection file created
     exit 0
 fi
 
 # Assume that there is no change if the saved hash is equal to the current one.
 # Otherwise assume that the files where changed and overwrite the hash with the current one for the next call
 if [[ $(< "${ARTIFACTS_CHANGE_DETECTION_HASH_FILE_PATH}") == "$CURRENT_ARTIFACTS_HASH" ]] ; then
-    echo 0
+    echo 0 # 0=No change detected
 else
     if ! ${readonlyMode}; then
         # Write the updated hash into the file containing the hash of the files list for the next call
         echo "$CURRENT_ARTIFACTS_HASH" > "${ARTIFACTS_CHANGE_DETECTION_HASH_FILE_PATH}"
         echo "detectChangedArtifacts: Change detection file updated" >&2
     fi
-    echo 2
+    echo 2 # 2=Change detected and change detection file updated
 fi
diff --git a/scripts/downloader/downloadTypescriptProject.sh b/scripts/downloader/downloadTypescriptProject.sh
@@ -177,9 +177,4 @@ fi
 (
   cd "${fullSourceDirectory}" || exit
   usePackageManagerToInstallDependencies
-  echo "downloadTypescriptProject: Scanning Typescript source using @jqassistant/ts-lce..."
-  npx --yes @jqassistant/ts-lce@1.2.0 --extension React >"./../../runtime/logs/jqassistant-typescript-scan-${projectName}.log" 2>&1 || exit
-)
-echo "downloadTypescriptProject: Moving scanned results into the artifacts/typescript directory..."
-mkdir -p artifacts/typescript
-mv -fv "${fullSourceDirectory}/.reports/jqa/ts-output.json" "artifacts/typescript/${fullProjectName}.json"
+)
diff --git a/scripts/findPathsToScan.sh b/scripts/findPathsToScan.sh
diff --git a/scripts/findTypescriptDataFiles.sh b/scripts/findTypescriptDataFiles.sh
diff --git a/scripts/importGit.sh b/scripts/importGit.sh
diff --git a/scripts/resetAndScan.sh b/scripts/resetAndScan.sh
diff --git a/scripts/resetAndScanChanged.sh b/scripts/resetAndScanChanged.sh

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+// Create index for git commit sha`
	`2`	`+`
	`3`	`+CREATE INDEX INDEX_COMMIT_HASH IF NOT EXISTS FOR (commit:Commit) ON (commit.sha)`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+// Create index for the relative file path`
	`2`	`+`
	`3`	`+CREATE INDEX INDEX_FILE_NAME IF NOT EXISTS FOR (file:File) ON (file.relativePath)`