Skip to content

Scan each contained Typescript package when the scan of the whole source repository failed #244

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 11, 2024
Merged

Scan each contained Typescript package when the scan of the whole source repository failed #244

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 81 additions & 17 deletions scripts/scanTypescript.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")"
echo "scanTypescript: SCRIPTS_DIR=${SCRIPTS_DIR}" >&2

# Dry run for internal testing (not intended to be accessible from the outside)
TYPESCRIPT_SCAN_DRY_RUN=false
TYPESCRIPT_SCAN_DRY_RUN=false
if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = true ] ; then
echo "scanTypescript: -> DRY RUN <- Scanning will only be logged, not executed." >&2
fi
Expand All @@ -39,6 +39,66 @@ if ! command -v "npx" &> /dev/null ; then
exit 1
fi

# Takes one parameter containing the directory to search.
# Returns all directories (multi-line) that contain a "package.json" file within the given base directory.
find_directories_with_package_json_file() {
find -L "${1}" \
-type d -name "node_modules" -prune -o \
-type d -name "dist" -prune -o \
-type d -name ".yalc" -prune -o \
-type d -name "lib" -prune -o \
-type d -name "libs" -prune -o \
-type d -name "*test" -prune -o \
-type d -name "*tests" -prune -o \
-name "package.json" \
-print0 \
| xargs -0 -r -I {} dirname {}
}

# Takes one parameter containing the directory to scan for Typescript projects and the second one containing progress information.
# Executes the Typescript scan for the given base directory including subdirectories.
# Skips the scan in case of a dry run
scan_directory() {
local source_directory_name; source_directory_name=$(basename "${1}");
local progress_information; progress_information="${2}"

echo "" >&2 # Output an empty line to have a clearer separation between each scan

if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = false ] ; then
echo "scanTypescript: $(date +'%Y-%m-%dT%H:%M:%S%z') Scanning ${source_directory_name} (${progress_information}) -----------------" >&2
# Note: For later troubleshooting, the output is also copied to a dedicated log file using "tee".
# Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager.
# Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for scanning larger projects
NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=${TYPESCRIPT_SCAN_HEAP_MEMORY}" npx --yes @jqassistant/[email protected] "${1}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2
else
echo "scanTypescript: Skipping scan of ${source_directory_name} (${progress_information}) -----------------" >&2
fi
}

# Takes one parameter containing the directory to scan for Typescript projects.
# Returns true (=0) when the given directory contains a valid (existing and reasonable size) scan result file.
# Otherwise returns false
is_valid_scan_result() {
if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = true ] ; then
echo "scanTypescript: Info: No scan result expected in dry run mode." >&2
return 1 # (false) Since dry run mode won't produce a scan result. Additionally, its intended to also dry-run the individual package scans.
fi

local scan_result_file="${1}/.reports/jqa/ts-output.json"
if [ ! -f "${scan_result_file}" ] ; then
echo "scanTypescript: Info: The scanned file ${scan_result_file} doesn't exist" >&2
return 1 # (false) Since the file doesn't exist it is considered empty.
fi

local scan_file_size; scan_file_size=$(wc -c "${scan_result_file}" | awk '{print $1}')
if [ "${scan_file_size}" -le "600" ]; then
echo "scanTypescript: Info: The scanned file ${scan_result_file} is too small: ${scan_file_size} < 600" >&2
false
else
true
fi
}

# Scan and analyze Artifacts when they were changed
changeDetectionHashFilePath="./${SOURCE_DIRECTORY}/typescriptFileChangeDetectionHashFile.txt"
changeDetectionReturnCode=$( source "${SCRIPTS_DIR}/detectChangedFiles.sh" --readonly --hashfile "${changeDetectionHashFilePath}" --paths "./${SOURCE_DIRECTORY}")
Expand All @@ -54,23 +114,27 @@ if [ "${changeDetectionReturnCode}" != "0" ] || [ "${TYPESCRIPT_SCAN_DRY_RUN}" =
LOG_DIRECTORY="$(pwd)/runtime/logs"
echo "scanTypescript: LOG_DIRECTORY=${LOG_DIRECTORY}" >&2

source_directories=$( find -L "./${SOURCE_DIRECTORY}" -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -r -I {} echo {} )

total_directories=$(echo "${source_directories}" | wc -l | awk '{print $1}')
processed_directories=0

for directory in ${source_directories}; do
directory_name=$(basename "${directory}");
processed_directories=$((processed_directories + 1))
echo "" >&2
echo "scanTypescript: $(date +'%Y-%m-%dT%H:%M:%S%z') Scanning ${directory_name} (${processed_directories}/${total_directories}) -----------------" >&2

if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = false ] ; then
# Note: For later troubleshooting, the output is also copied to a dedicated log file using "tee".
# Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager.
# Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for scanning larger projects
NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=${TYPESCRIPT_SCAN_HEAP_MEMORY}" npx --yes @jqassistant/[email protected] "${directory}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2
source_directories=$( find -L "./${SOURCE_DIRECTORY}" -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -r -I {} echo {} )
total_source_directories=$(echo "${source_directories}" | wc -l | awk '{print $1}')
processed_source_directories=0

for source_directory in ${source_directories}; do
processed_source_directories=$((processed_source_directories + 1))
progress_info_source_dirs="${processed_source_directories}/${total_source_directories}"
if scan_directory "${source_directory}" "${progress_info_source_dirs}" && is_valid_scan_result "${source_directory}"; then
continue # successful scan, proceed to next one.
fi

echo "scanTypescript: Info: Unsuccessful source directory scan. Trying to scan all contained packages individually." >&2
contained_package_directories=$( find_directories_with_package_json_file "${source_directory}" )
total_package_directories=$(echo "${contained_package_directories}" | wc -l | awk '{print $1}')
processed_package_directories=0

for contained_package_directory in ${contained_package_directories}; do
processed_package_directories=$((processed_package_directories + 1))
progress_info_package_dirs="${progress_info_source_dirs}: ${processed_package_directories}/${total_package_directories}"
scan_directory "${contained_package_directory}" "${progress_info_package_dirs}"
done
done

if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = false ] ; then
Expand Down
Loading