diff --git a/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher b/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher index c341ae58f..26bfc93a2 100644 --- a/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher +++ b/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher @@ -17,7 +17,7 @@ MATCH (npmPackage:NPM:Package) ,collect(DISTINCT project) AS projects ,collect(DISTINCT projectConfigDir) AS projectConfigDirs // Assure that the found connection is unique and not ambiguous - WHERE size(projects) = 1 + WHERE size(projects) >= 1 // might appear more than once when there are multiple tsconfig files AND size(projectConfigDirs) = 1 UNWIND projects AS project // Create a HAS_NPM_PACKAGE relationship between the Typescript project and the npm package @@ -29,4 +29,4 @@ MATCH (npmPackage:NPM:Package) // Detailed results for debugging //RETURN npmPackage.fileName AS npmPackageFileName // ,projectConfigDirs[0].absoluteFileName AS projectConfigDirectory -// ,relativeNpmPackageDirectory \ No newline at end of file +// ,relativeNpmPackageDirectory diff --git a/scripts/copyPackageJsonFiles.sh b/scripts/copyPackageJsonFiles.sh index 57ad91a9f..92b836445 100755 --- a/scripts/copyPackageJsonFiles.sh +++ b/scripts/copyPackageJsonFiles.sh @@ -19,6 +19,21 @@ if [ ! -d "${SOURCE_DIRECTORY}" ]; then return 0 fi +# Returns all relevant package.json files for the source directory given as first and only parameter. +find_package_json_files() { + find -L "${1}" \ + -type d -name "node_modules" -prune -o \ + -type d -name "dist" -prune -o \ + -type d -name ".yalc" -prune -o \ + -type d -name "target" -prune -o \ + -type d -name "temp" -prune -o \ + -type d -name "lib" -prune -o \ + -type d -name "libs" -prune -o \ + -name 'package.json' \ + -print0 | \ + xargs -0 -r -I {} echo {} +} + ( cd "./${SOURCE_DIRECTORY}" @@ -27,8 +42,7 @@ fi copied_package_json_files=0 - #for file in $( find -L . -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {}); do - for file in $( find -L . -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {} echo {}); do + for file in $( find_package_json_files . ); do fileDirectory=$(dirname "${file}") targetDirectory="../${ARTIFACTS_DIRECTORY}/${NPM_PACKAGE_JSON_ARTIFACTS_DIRECTORY}/${fileDirectory}" # echo "copyPackageJsonFiles: Debug: Copying ${file} to ${targetDirectory}" # debug logging diff --git a/scripts/findPathsToScan.sh b/scripts/findPathsToScan.sh index 0af6f2515..6ff734b1e 100755 --- a/scripts/findPathsToScan.sh +++ b/scripts/findPathsToScan.sh @@ -40,8 +40,9 @@ fi if [ -d "./${SOURCE_DIRECTORY}" ] ; then # Scan Typescript analysis json data files in the source directory - typescriptAnalysisFiles="$(find "./${SOURCE_DIRECTORY}" \ + typescriptAnalysisFiles="$(find -L "./${SOURCE_DIRECTORY}" \ -type d -name "node_modules" -prune -o \ + -type d -name "dist" -prune -o \ -type d -name "target" -prune -o \ -type d -name "temp" -prune -o \ -type f -path "*/.reports/jqa/ts-output.json" \ @@ -54,15 +55,16 @@ if [ -d "./${SOURCE_DIRECTORY}" ] ; then # Scan package.json files for npm (nodes package manager) in the source directory # # TODO The following lines can be reactivated when the following issue is resolved: # https://github.com/jqassistant-plugin/jqassistant-npm-plugin/issues/5 - #npmPackageJsonFiles="$(find "${SOURCE_DIRECTORY}" -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {} | tr '\n' ',' | sed 's/,$/\n/')" + #npmPackageJsonFiles="$(find -L "${SOURCE_DIRECTORY}" -type d -name node_modules -prune -o -name 'package.json' -print0 | xargs -0 -r -I {} | tr '\n' ',' | sed 's/,$/\n/')" #if [ -n "${npmPackageJsonFiles}" ]; then # directoriesAndFilesToScan="$(appendNonEmpty "${directoriesAndFilesToScan}")${npmPackageJsonFiles}" #fi # Scan git repositories in the artifacts directory if [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "" ] || [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "plugin" ] ; then - gitDirectories="$(find "./${SOURCE_DIRECTORY}" \ + gitDirectories="$(find -L "./${SOURCE_DIRECTORY}" \ -type d -name "node_modules" -prune -o \ + -type d -name "dist" -prune -o \ -type d -name "target" -prune -o \ -type d -name "temp" -prune -o \ -type d -name ".git" -exec echo {} \; | tr '\n' ',' | sed 's/,$/\n/')" diff --git a/scripts/importGit.sh b/scripts/importGit.sh index 44c19911f..d6466279f 100755 --- a/scripts/importGit.sh +++ b/scripts/importGit.sh @@ -175,7 +175,7 @@ if [ ! "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "none" ] && [ ! "${IMPORT existing_data_has_been_deleted=false - for repository in $(find "${source}" -type d -name ".git" -print0 | xargs -0 -r -I {} dirname {}); do + for repository in $(find -L "${source}" -type d -name ".git" -print0 | xargs -0 -r -I {} dirname {}); do # Prepare import by cleaning existing data first if [ "${existing_data_has_been_deleted}" = false ] ; then deleteExistingGitData diff --git a/scripts/scanTypescript.sh b/scripts/scanTypescript.sh index d967c75b7..a7080040e 100755 --- a/scripts/scanTypescript.sh +++ b/scripts/scanTypescript.sh @@ -37,6 +37,8 @@ else echo "scanTypescript: Detected change (${changeDetectionReturnCode}). Scanning Typescript source using @jqassistant/ts-lce." mkdir -p "./runtime/logs" + LOG_DIRECTORY="$(pwd)/runtime/logs" + echo "scanTypescript: LOG_DIRECTORY=${LOG_DIRECTORY}" >&2 source_directories=$( find -L "./${SOURCE_DIRECTORY}" -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -r -I {} echo {} ) total_directories=$(echo "${source_directories}" | wc -l | awk '{print $1}') @@ -44,18 +46,15 @@ else for directory in ${source_directories}; do directory_name=$(basename "${directory}"); - # Note: The npx command will be executed in the source directory using a subshell by putting parentheses around it. - # The subshell is the reason why it isn't needed to change back to the main directory after execution. + processed_directories=$((processed_directories + 1)) + echo "" >&2 + echo "scanTypescript: $(date +'%Y-%m-%dT%H:%M:%S%z') Scanning ${directory_name} (${processed_directories}/${total_directories}) -----------------" >&2 # Note: This script must not output anything except for the return code to stdout, # all output of the scanning needs to be redirected to stderr using ">&2". # For later troubleshooting, the output is also copied to a dedicated log file using "tee". # Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager. # Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for larger projects to scan - ( cd "${directory}" && NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=4096" npx --yes @jqassistant/ts-lce@1.3.0 --extension React 2>&1 | tee "./../../runtime/logs/jqassistant-typescript-scan-${directory_name}.log" >&2 || exit ) - processed_directories=$((processed_directories + 1)) - echo "" >&2 - echo "scanTypescript: Scanned ${directory_name} (${processed_directories}/${total_directories}) ----------------------" >&2 - echo "" >&2 + NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=4096" npx --yes @jqassistant/ts-lce@1.3.0 "${directory}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2 done changeDetectionReturnCode=$( source "${SCRIPTS_DIR}/detectChangedFiles.sh" --hashfile "${changeDetectionHashFilePath}" --paths "./${SOURCE_DIRECTORY}")