Merge pull request #244 from JohT/feature/scan-each-package-when-repo-scan-failed

Scan each contained Typescript package when the scan of the whole source repository failed

Author: JohT

scripts/scanTypescript.sh

@@ -24,7 +24,7 @@ SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")"
 echo "scanTypescript: SCRIPTS_DIR=${SCRIPTS_DIR}" >&2
 
 # Dry run for internal testing (not intended to be accessible from the outside)
-TYPESCRIPT_SCAN_DRY_RUN=false
+TYPESCRIPT_SCAN_DRY_RUN=false 
 if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = true ] ; then
     echo "scanTypescript: -> DRY RUN <- Scanning will only be logged, not executed." >&2
 fi
@@ -39,6 +39,66 @@ if ! command -v "npx" &> /dev/null ; then
     exit 1
 fi
 
+# Takes one parameter containing the directory to search.
+# Returns all directories (multi-line) that contain a "package.json" file within the given base directory.
+find_directories_with_package_json_file() {
+    find -L "${1}" \
+        -type d -name "node_modules" -prune -o \
+        -type d -name "dist" -prune -o \
+        -type d -name ".yalc" -prune -o \
+        -type d -name "lib" -prune -o \
+        -type d -name "libs" -prune -o \
+        -type d -name "*test" -prune -o \
+        -type d -name "*tests" -prune -o \
+        -name "package.json" \
+        -print0 \
+        | xargs -0 -r -I {} dirname {}
+}
+
+# Takes one parameter containing the directory to scan for Typescript projects and the second one containing progress information.
+# Executes the Typescript scan for the given base directory including subdirectories.
+# Skips the scan in case of a dry run
+scan_directory() {
+    local source_directory_name; source_directory_name=$(basename "${1}");
+    local progress_information; progress_information="${2}"
+
+    echo "" >&2 # Output an empty line to have a clearer separation between each scan
+    
+    if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = false ] ; then
+        echo "scanTypescript: $(date +'%Y-%m-%dT%H:%M:%S%z') Scanning ${source_directory_name} (${progress_information}) -----------------" >&2
+        # Note: For later troubleshooting, the output is also copied to a dedicated log file using "tee".
+        # Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager.
+        # Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for scanning larger projects
+        NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=${TYPESCRIPT_SCAN_HEAP_MEMORY}" npx --yes @jqassistant/[email protected] "${1}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2
+    else
+        echo "scanTypescript: Skipping scan of ${source_directory_name} (${progress_information}) -----------------" >&2
+    fi
+}
+
+# Takes one parameter containing the directory to scan for Typescript projects.
+# Returns true (=0) when the given directory contains a valid (existing and reasonable size) scan result file.
+# Otherwise returns false
+is_valid_scan_result() {
+    if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = true ] ; then
+        echo "scanTypescript: Info: No scan result expected in dry run mode." >&2
+        return 1 # (false) Since dry run mode won't produce a scan result. Additionally, its intended to also dry-run the individual package scans.
+    fi
+
+    local scan_result_file="${1}/.reports/jqa/ts-output.json"
+    if [ ! -f "${scan_result_file}" ] ; then
+        echo "scanTypescript: Info: The scanned file ${scan_result_file} doesn't exist" >&2
+        return 1 # (false) Since the file doesn't exist it is considered empty.
+    fi
+
+    local scan_file_size; scan_file_size=$(wc -c "${scan_result_file}" | awk '{print $1}')
+    if [ "${scan_file_size}" -le "600" ]; then
+        echo "scanTypescript: Info: The scanned file ${scan_result_file} is too small: ${scan_file_size} < 600" >&2
+        false
+    else
+        true
+    fi
+}
+
 # Scan and analyze Artifacts when they were changed
 changeDetectionHashFilePath="./${SOURCE_DIRECTORY}/typescriptFileChangeDetectionHashFile.txt"
 changeDetectionReturnCode=$( source "${SCRIPTS_DIR}/detectChangedFiles.sh" --readonly --hashfile "${changeDetectionHashFilePath}" --paths "./${SOURCE_DIRECTORY}")
@@ -54,23 +114,27 @@ if [ "${changeDetectionReturnCode}" != "0" ] || [ "${TYPESCRIPT_SCAN_DRY_RUN}" =
     LOG_DIRECTORY="$(pwd)/runtime/logs"
     echo "scanTypescript: LOG_DIRECTORY=${LOG_DIRECTORY}" >&2
 
-    source_directories=$( find -L "./${SOURCE_DIRECTORY}" -mindepth 1 -maxdepth 1 -type d  -print0 | xargs -0 -r -I {} echo {} )
-    
-    total_directories=$(echo "${source_directories}" | wc -l | awk '{print $1}')
-    processed_directories=0
-
-    for directory in ${source_directories}; do
-        directory_name=$(basename "${directory}");
-        processed_directories=$((processed_directories + 1))
-        echo "" >&2
-        echo "scanTypescript: $(date +'%Y-%m-%dT%H:%M:%S%z') Scanning ${directory_name} (${processed_directories}/${total_directories}) -----------------" >&2
-        
-        if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = false ] ; then
-            # Note: For later troubleshooting, the output is also copied to a dedicated log file using "tee".
-            # Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager.
-            # Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for scanning larger projects
-            NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=${TYPESCRIPT_SCAN_HEAP_MEMORY}" npx --yes @jqassistant/[email protected] "${directory}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2
+    source_directories=$( find -L "./${SOURCE_DIRECTORY}" -mindepth 1 -maxdepth 1 -type d  -print0 | xargs -0 -r -I {} echo {} )   
+    total_source_directories=$(echo "${source_directories}" | wc -l | awk '{print $1}')
+    processed_source_directories=0
+
+    for source_directory in ${source_directories}; do
+        processed_source_directories=$((processed_source_directories + 1))
+        progress_info_source_dirs="${processed_source_directories}/${total_source_directories}"
+        if scan_directory "${source_directory}" "${progress_info_source_dirs}" && is_valid_scan_result "${source_directory}"; then
+            continue # successful scan, proceed to next one.
         fi
+
+        echo "scanTypescript: Info: Unsuccessful source directory scan. Trying to scan all contained packages individually." >&2
+        contained_package_directories=$( find_directories_with_package_json_file "${source_directory}" )
+        total_package_directories=$(echo "${contained_package_directories}" | wc -l | awk '{print $1}')
+        processed_package_directories=0
+
+        for contained_package_directory in ${contained_package_directories}; do
+            processed_package_directories=$((processed_package_directories + 1))
+            progress_info_package_dirs="${progress_info_source_dirs}: ${processed_package_directories}/${total_package_directories}"
+            scan_directory "${contained_package_directory}" "${progress_info_package_dirs}"
+        done
     done
 
     if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = false ] ; then