Merge pull request #229 from JohT/fix/scan-large-typscript-code-bases

Support scanning large code bases

Author: JohT

cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher

@@ -17,7 +17,7 @@ MATCH (npmPackage:NPM:Package)
       ,collect(DISTINCT project) AS projects
       ,collect(DISTINCT projectConfigDir) AS projectConfigDirs
 // Assure that the found connection is unique and not ambiguous 
- WHERE size(projects)          = 1
+ WHERE size(projects)          >= 1 // might appear more than once when there are multiple tsconfig files
    AND size(projectConfigDirs) = 1
  UNWIND projects AS project
 // Create a HAS_NPM_PACKAGE relationship between the Typescript project and the npm package
@@ -29,4 +29,4 @@ MATCH (npmPackage:NPM:Package)
 // Detailed results for debugging
 //RETURN npmPackage.fileName                   AS npmPackageFileName
 //      ,projectConfigDirs[0].absoluteFileName AS projectConfigDirectory
-//      ,relativeNpmPackageDirectory
+//      ,relativeNpmPackageDirectory

scripts/copyPackageJsonFiles.sh

@@ -19,6 +19,21 @@ if [ ! -d "${SOURCE_DIRECTORY}" ]; then
   return 0
 fi
 
+# Returns all relevant package.json files for the source directory given as first and only parameter. 
+find_package_json_files() {
+  find -L "${1}" \
+      -type d -name "node_modules" -prune -o \
+      -type d -name "dist" -prune -o \
+      -type d -name ".yalc" -prune -o \
+      -type d -name "target" -prune -o \
+      -type d -name "temp" -prune -o \
+      -type d -name "lib" -prune -o \
+      -type d -name "libs" -prune -o \
+      -name 'package.json' \
+      -print0 | \
+      xargs -0 -r -I {} echo {}
+}
+
 (
     cd "./${SOURCE_DIRECTORY}"
 
@@ -27,8 +42,7 @@ fi
 
     copied_package_json_files=0
 
-   #for file in $( find -L . -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {}); do
-    for file in $( find -L . -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {} echo {}); do
+    for file in $( find_package_json_files . ); do
         fileDirectory=$(dirname "${file}")
         targetDirectory="../${ARTIFACTS_DIRECTORY}/${NPM_PACKAGE_JSON_ARTIFACTS_DIRECTORY}/${fileDirectory}"
         # echo "copyPackageJsonFiles: Debug: Copying ${file} to ${targetDirectory}" # debug logging

scripts/findPathsToScan.sh

@@ -40,8 +40,9 @@ fi
 
 if [ -d "./${SOURCE_DIRECTORY}" ] ; then
     # Scan Typescript analysis json data files in the source directory
-    typescriptAnalysisFiles="$(find "./${SOURCE_DIRECTORY}" \
+    typescriptAnalysisFiles="$(find -L "./${SOURCE_DIRECTORY}" \
                                     -type d -name "node_modules" -prune -o \
+                                    -type d -name "dist" -prune -o \
                                     -type d -name "target" -prune -o \
                                     -type d -name "temp" -prune -o \
                                     -type f -path "*/.reports/jqa/ts-output.json" \
@@ -54,15 +55,16 @@ if [ -d "./${SOURCE_DIRECTORY}" ] ; then
     # Scan package.json files for npm (nodes package manager) in the source directory
     # # TODO The following lines can be reactivated when the following issue is resolved:
     # https://github.com/jqassistant-plugin/jqassistant-npm-plugin/issues/5
-    #npmPackageJsonFiles="$(find "${SOURCE_DIRECTORY}" -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {} | tr '\n' ',' | sed 's/,$/\n/')"
+    #npmPackageJsonFiles="$(find -L "${SOURCE_DIRECTORY}" -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {} | tr '\n' ',' | sed 's/,$/\n/')"
     #if [ -n "${npmPackageJsonFiles}" ]; then
     #    directoriesAndFilesToScan="$(appendNonEmpty "${directoriesAndFilesToScan}")${npmPackageJsonFiles}"
     #fi
 
     # Scan git repositories in the artifacts directory
     if [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "" ] || [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "plugin" ] ; then
-        gitDirectories="$(find "./${SOURCE_DIRECTORY}" \
+        gitDirectories="$(find -L "./${SOURCE_DIRECTORY}" \
                                 -type d -name "node_modules" -prune -o \
+                                -type d -name "dist" -prune -o \
                                 -type d -name "target" -prune -o \
                                 -type d -name "temp" -prune -o \
                                 -type d -name ".git" -exec echo {} \; | tr '\n' ',' | sed 's/,$/\n/')"

scripts/importGit.sh

@@ -175,7 +175,7 @@ if [ ! "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "none" ] && [ ! "${IMPORT
 
   existing_data_has_been_deleted=false
 
-  for repository in $(find "${source}" -type d -name ".git" -print0 | xargs -0 -r -I {} dirname {}); do
+  for repository in $(find -L "${source}" -type d -name ".git" -print0 | xargs -0 -r -I {} dirname {}); do
     # Prepare import by cleaning existing data first
     if [ "${existing_data_has_been_deleted}" = false ] ; then
       deleteExistingGitData

scripts/scanTypescript.sh

@@ -37,25 +37,24 @@ else
     echo "scanTypescript: Detected change (${changeDetectionReturnCode}). Scanning Typescript source using @jqassistant/ts-lce."
 
     mkdir -p "./runtime/logs"
+    LOG_DIRECTORY="$(pwd)/runtime/logs"
+    echo "scanTypescript: LOG_DIRECTORY=${LOG_DIRECTORY}" >&2
 
     source_directories=$( find -L "./${SOURCE_DIRECTORY}" -mindepth 1 -maxdepth 1 -type d  -print0 | xargs -0 -r -I {} echo {} )
     total_directories=$(echo "${source_directories}" | wc -l | awk '{print $1}')
     processed_directories=0
 
     for directory in ${source_directories}; do
         directory_name=$(basename "${directory}");
-        # Note: The npx command will be executed in the source directory using a subshell by putting parentheses around it.
-        #       The subshell is the reason why it isn't needed to change back to the main directory after execution.
+        processed_directories=$((processed_directories + 1))
+        echo "" >&2
+        echo "scanTypescript: $(date +'%Y-%m-%dT%H:%M:%S%z') Scanning ${directory_name} (${processed_directories}/${total_directories}) -----------------" >&2
         # Note: This script must not output anything except for the return code to stdout,
         #       all output of the scanning needs to be redirected to stderr using ">&2".
         #       For later troubleshooting, the output is also copied to a dedicated log file using "tee".
         # Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager.
         # Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for larger projects to scan
-        ( cd "${directory}" && NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=4096" npx --yes @jqassistant/[email protected] --extension React 2>&1 | tee "./../../runtime/logs/jqassistant-typescript-scan-${directory_name}.log" >&2 || exit )
-        processed_directories=$((processed_directories + 1))
-        echo "" >&2
-        echo "scanTypescript: Scanned ${directory_name} (${processed_directories}/${total_directories}) ----------------------" >&2
-        echo "" >&2
+        NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=4096" npx --yes @jqassistant/[email protected] "${directory}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2
     done
 
     changeDetectionReturnCode=$( source "${SCRIPTS_DIR}/detectChangedFiles.sh" --hashfile "${changeDetectionHashFilePath}" --paths "./${SOURCE_DIRECTORY}")