From 87dfb19bd042a0eaacd0d64d8cd469a7106c8909 Mon Sep 17 00:00:00 2001 From: JohT <7671054+JohT@users.noreply.github.com> Date: Mon, 16 Sep 2024 17:26:03 +0200 Subject: [PATCH 1/5] Determine log directory before Typescript scanning. --- scripts/scanTypescript.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/scanTypescript.sh b/scripts/scanTypescript.sh index d967c75b7..dbf24ba0b 100755 --- a/scripts/scanTypescript.sh +++ b/scripts/scanTypescript.sh @@ -37,6 +37,8 @@ else echo "scanTypescript: Detected change (${changeDetectionReturnCode}). Scanning Typescript source using @jqassistant/ts-lce." mkdir -p "./runtime/logs" + LOG_DIRECTORY="$(pwd)/runtime/logs" + echo "scanTypescript: LOG_DIRECTORY=${LOG_DIRECTORY}" >&2 source_directories=$( find -L "./${SOURCE_DIRECTORY}" -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -r -I {} echo {} ) total_directories=$(echo "${source_directories}" | wc -l | awk '{print $1}') @@ -51,7 +53,7 @@ else # For later troubleshooting, the output is also copied to a dedicated log file using "tee". # Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager. # Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for larger projects to scan - ( cd "${directory}" && NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=4096" npx --yes @jqassistant/ts-lce@1.3.0 --extension React 2>&1 | tee "./../../runtime/logs/jqassistant-typescript-scan-${directory_name}.log" >&2 || exit ) + ( cd "${directory}" && NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=4096" npx --yes @jqassistant/ts-lce@1.3.0 --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2 || exit ) processed_directories=$((processed_directories + 1)) echo "" >&2 echo "scanTypescript: Scanned ${directory_name} (${processed_directories}/${total_directories}) ----------------------" >&2 From f38ccd0c5eacbd1d6db660cca02808454a0ccdc2 Mon Sep 17 00:00:00 2001 From: JohT <7671054+JohT@users.noreply.github.com> Date: Mon, 16 Sep 2024 17:28:56 +0200 Subject: [PATCH 2/5] Run Typescript scan in workspace and pass scan directory --- scripts/scanTypescript.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/scanTypescript.sh b/scripts/scanTypescript.sh index dbf24ba0b..e8317cd69 100755 --- a/scripts/scanTypescript.sh +++ b/scripts/scanTypescript.sh @@ -46,14 +46,12 @@ else for directory in ${source_directories}; do directory_name=$(basename "${directory}"); - # Note: The npx command will be executed in the source directory using a subshell by putting parentheses around it. - # The subshell is the reason why it isn't needed to change back to the main directory after execution. # Note: This script must not output anything except for the return code to stdout, # all output of the scanning needs to be redirected to stderr using ">&2". # For later troubleshooting, the output is also copied to a dedicated log file using "tee". # Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager. # Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for larger projects to scan - ( cd "${directory}" && NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=4096" npx --yes @jqassistant/ts-lce@1.3.0 --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2 || exit ) + NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=4096" npx --yes @jqassistant/ts-lce@1.3.0 "${directory}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2 processed_directories=$((processed_directories + 1)) echo "" >&2 echo "scanTypescript: Scanned ${directory_name} (${processed_directories}/${total_directories}) ----------------------" >&2 From 24797e8e2b477faa71d8e342869a65a12166cc8c Mon Sep 17 00:00:00 2001 From: JohT <7671054+JohT@users.noreply.github.com> Date: Mon, 16 Sep 2024 17:43:32 +0200 Subject: [PATCH 3/5] Log start of Typescript scan with timestamp. --- scripts/scanTypescript.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/scanTypescript.sh b/scripts/scanTypescript.sh index e8317cd69..a7080040e 100755 --- a/scripts/scanTypescript.sh +++ b/scripts/scanTypescript.sh @@ -46,16 +46,15 @@ else for directory in ${source_directories}; do directory_name=$(basename "${directory}"); + processed_directories=$((processed_directories + 1)) + echo "" >&2 + echo "scanTypescript: $(date +'%Y-%m-%dT%H:%M:%S%z') Scanning ${directory_name} (${processed_directories}/${total_directories}) -----------------" >&2 # Note: This script must not output anything except for the return code to stdout, # all output of the scanning needs to be redirected to stderr using ">&2". # For later troubleshooting, the output is also copied to a dedicated log file using "tee". # Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager. # Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for larger projects to scan NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=4096" npx --yes @jqassistant/ts-lce@1.3.0 "${directory}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2 - processed_directories=$((processed_directories + 1)) - echo "" >&2 - echo "scanTypescript: Scanned ${directory_name} (${processed_directories}/${total_directories}) ----------------------" >&2 - echo "" >&2 done changeDetectionReturnCode=$( source "${SCRIPTS_DIR}/detectChangedFiles.sh" --hashfile "${changeDetectionHashFilePath}" --paths "./${SOURCE_DIRECTORY}") From 3cc1e8b90326eafd8fcd9e9f35a871b34776ba51 Mon Sep 17 00:00:00 2001 From: JohT <7671054+JohT@users.noreply.github.com> Date: Mon, 16 Sep 2024 17:44:42 +0200 Subject: [PATCH 4/5] Filter relevant package.json files --- scripts/copyPackageJsonFiles.sh | 18 ++++++++++++++++-- scripts/findPathsToScan.sh | 8 +++++--- scripts/importGit.sh | 2 +- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/scripts/copyPackageJsonFiles.sh b/scripts/copyPackageJsonFiles.sh index 57ad91a9f..92b836445 100755 --- a/scripts/copyPackageJsonFiles.sh +++ b/scripts/copyPackageJsonFiles.sh @@ -19,6 +19,21 @@ if [ ! -d "${SOURCE_DIRECTORY}" ]; then return 0 fi +# Returns all relevant package.json files for the source directory given as first and only parameter. +find_package_json_files() { + find -L "${1}" \ + -type d -name "node_modules" -prune -o \ + -type d -name "dist" -prune -o \ + -type d -name ".yalc" -prune -o \ + -type d -name "target" -prune -o \ + -type d -name "temp" -prune -o \ + -type d -name "lib" -prune -o \ + -type d -name "libs" -prune -o \ + -name 'package.json' \ + -print0 | \ + xargs -0 -r -I {} echo {} +} + ( cd "./${SOURCE_DIRECTORY}" @@ -27,8 +42,7 @@ fi copied_package_json_files=0 - #for file in $( find -L . -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {}); do - for file in $( find -L . -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {} echo {}); do + for file in $( find_package_json_files . ); do fileDirectory=$(dirname "${file}") targetDirectory="../${ARTIFACTS_DIRECTORY}/${NPM_PACKAGE_JSON_ARTIFACTS_DIRECTORY}/${fileDirectory}" # echo "copyPackageJsonFiles: Debug: Copying ${file} to ${targetDirectory}" # debug logging diff --git a/scripts/findPathsToScan.sh b/scripts/findPathsToScan.sh index 0af6f2515..6ff734b1e 100755 --- a/scripts/findPathsToScan.sh +++ b/scripts/findPathsToScan.sh @@ -40,8 +40,9 @@ fi if [ -d "./${SOURCE_DIRECTORY}" ] ; then # Scan Typescript analysis json data files in the source directory - typescriptAnalysisFiles="$(find "./${SOURCE_DIRECTORY}" \ + typescriptAnalysisFiles="$(find -L "./${SOURCE_DIRECTORY}" \ -type d -name "node_modules" -prune -o \ + -type d -name "dist" -prune -o \ -type d -name "target" -prune -o \ -type d -name "temp" -prune -o \ -type f -path "*/.reports/jqa/ts-output.json" \ @@ -54,15 +55,16 @@ if [ -d "./${SOURCE_DIRECTORY}" ] ; then # Scan package.json files for npm (nodes package manager) in the source directory # # TODO The following lines can be reactivated when the following issue is resolved: # https://github.com/jqassistant-plugin/jqassistant-npm-plugin/issues/5 - #npmPackageJsonFiles="$(find "${SOURCE_DIRECTORY}" -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {} | tr '\n' ',' | sed 's/,$/\n/')" + #npmPackageJsonFiles="$(find -L "${SOURCE_DIRECTORY}" -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {} | tr '\n' ',' | sed 's/,$/\n/')" #if [ -n "${npmPackageJsonFiles}" ]; then # directoriesAndFilesToScan="$(appendNonEmpty "${directoriesAndFilesToScan}")${npmPackageJsonFiles}" #fi # Scan git repositories in the artifacts directory if [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "" ] || [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "plugin" ] ; then - gitDirectories="$(find "./${SOURCE_DIRECTORY}" \ + gitDirectories="$(find -L "./${SOURCE_DIRECTORY}" \ -type d -name "node_modules" -prune -o \ + -type d -name "dist" -prune -o \ -type d -name "target" -prune -o \ -type d -name "temp" -prune -o \ -type d -name ".git" -exec echo {} \; | tr '\n' ',' | sed 's/,$/\n/')" diff --git a/scripts/importGit.sh b/scripts/importGit.sh index 44c19911f..d6466279f 100755 --- a/scripts/importGit.sh +++ b/scripts/importGit.sh @@ -175,7 +175,7 @@ if [ ! "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "none" ] && [ ! "${IMPORT existing_data_has_been_deleted=false - for repository in $(find "${source}" -type d -name ".git" -print0 | xargs -0 -r -I {} dirname {}); do + for repository in $(find -L "${source}" -type d -name ".git" -print0 | xargs -0 -r -I {} dirname {}); do # Prepare import by cleaning existing data first if [ "${existing_data_has_been_deleted}" = false ] ; then deleteExistingGitData From 171bea3155926ba600db1a6dfcf3aef40b94218f Mon Sep 17 00:00:00 2001 From: Johannes <7671054+JohT@users.noreply.github.com> Date: Wed, 18 Sep 2024 07:56:52 +0200 Subject: [PATCH 5/5] Support Typescript projects with multiple tsconfig files --- .../Link_projects_to_npm_packages.cypher | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher b/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher index c341ae58f..26bfc93a2 100644 --- a/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher +++ b/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher @@ -17,7 +17,7 @@ MATCH (npmPackage:NPM:Package) ,collect(DISTINCT project) AS projects ,collect(DISTINCT projectConfigDir) AS projectConfigDirs // Assure that the found connection is unique and not ambiguous - WHERE size(projects) = 1 + WHERE size(projects) >= 1 // might appear more than once when there are multiple tsconfig files AND size(projectConfigDirs) = 1 UNWIND projects AS project // Create a HAS_NPM_PACKAGE relationship between the Typescript project and the npm package @@ -29,4 +29,4 @@ MATCH (npmPackage:NPM:Package) // Detailed results for debugging //RETURN npmPackage.fileName AS npmPackageFileName // ,projectConfigDirs[0].absoluteFileName AS projectConfigDirectory -// ,relativeNpmPackageDirectory \ No newline at end of file +// ,relativeNpmPackageDirectory