diff --git a/COMMANDS.md b/COMMANDS.md index c02c517ad..f0f133a40 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -71,7 +71,9 @@ The [analyze.sh](./scripts/analysis/analyze.sh) command comes with these command - `--report Csv` only generates CSV reports. This speeds up the report generation and doesn't depend on Python, Jupyter Notebook or any other related dependencies. The default value os `All` to generate all reports. `Jupiter` will only generate Jupyter Notebook reports. `DatabaseCsvExport` exports the whole graph database as a CSV file (performance intense, check if there are security concerns first). - `--profile Neo4jv4` uses the older long term support (june 2023) version v4.4.x of Neo4j and suitable compatible versions of plugins and JQAssistant. `Neo4jv5` will explicitly select the newest (june 2023) version 5.x of Neo4j. Without setting -a profile, the newest versions will be used. Profiles are scripts that can be found in the directory [scripts/profiles](./scripts/profiles/). +a profile, the newest versions will be used. Profiles can be found in the directory [scripts/profiles](./scripts/profiles/). + +- `--profile Neo4jv5-continue-on-scan-errors` is based on the default profile (`Neo4jv5`) but uses the jQAssistant configuration template [template-neo4jv5-jqassistant-continue-on-error.yaml](./scripts/configuration/template-neo4jv5-jqassistant-continue-on-error.yaml) to continue on scan error instead of failing fast. This is temporarily useful when there is a known error that needs to be ignored. It is still recommended to use the default profile and fail fast if there is something wrong. Profiles can be found in the directory [scripts/profiles](./scripts/profiles/). - `--explore` activates the "explore" mode where no reports are generated. Furthermore, Neo4j won't be stopped at the end of the script and will therefore continue running. This makes it easy to just set everything up but then use the running Neo4j server to explore the data manually. diff --git a/README.md b/README.md index 1051dc893..ecea8dba9 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,14 @@ The [Code Structure Analysis Pipeline](./.github/workflows/java-code-analysis.ym TYPESCRIPT_SCAN_HEAP_MEMORY=16384 ./../../scripts/analysis/analyze.sh ``` +- How can i continue on errors when scanning Typescript projects instead of cancelling the whole analysis? + 👉 Use the profile `Neo4jv5-continue-on-scan-errors` (default = `Neo4jv5`): + + ```shell + ./../../scripts/analysis/analyze.sh --profile Neo4jv5-continue-on-scan-errors + ``` + + ## 🕸 Web References - [Graph Data Science 101: Understanding Graphs and Graph Data Science](https://techfirst.medium.com/graph-data-science-101-understanding-graphs-and-graph-data-science-c25055a9db01) diff --git a/cypher/Artifact_Dependencies/Set_maven_artifact_version.cypher b/cypher/Artifact_Dependencies/Set_maven_artifact_version.cypher new file mode 100644 index 000000000..5b02cedf8 --- /dev/null +++ b/cypher/Artifact_Dependencies/Set_maven_artifact_version.cypher @@ -0,0 +1,14 @@ +// Set property "version" on Artifact nodes to what is specified in the Maven POM + + MATCH (artifact:Java:Artifact)-[:CONTAINS]-(pom:Maven:Pom) + MATCH (pom)-[:HAS_ROOT_ELEMENT]->(pomProject:Element{name:'project'}) +OPTIONAL MATCH (pomProject)-[:HAS_ELEMENT]->(:Element{name:'version'})-[:HAS_TEXT]->(versionValue:Text) +OPTIONAL MATCH (pomProject)-[:HAS_ELEMENT]->(:Element{name:'parent'}) +-[:HAS_ELEMENT]->(:Element{name:'version'})-[:HAS_TEXT]->(parentVersionValue:Text) + WITH *, artifact, coalesce(versionValue.value, parentVersionValue.value) AS versionNumber + WHERE versionNumber IS NOT NULL + SET artifact.version = versionNumber + RETURN count(DISTINCT artifact.fileName) AS numberOfUpdatedArtifacts +// Debugging +// RETURN artifact.name, versionNumber, versionValue.value, parentVersionValue.value +// LIMIT 25 \ No newline at end of file diff --git a/cypher/Internal_Dependencies/Inter_scan_and_project_dependencies_of_Typescript_modules.cypher b/cypher/Internal_Dependencies/Inter_scan_and_project_dependencies_of_Typescript_modules.cypher new file mode 100644 index 000000000..95183d975 --- /dev/null +++ b/cypher/Internal_Dependencies/Inter_scan_and_project_dependencies_of_Typescript_modules.cypher @@ -0,0 +1,12 @@ +// Inter scan/project higher-level module dependencies for manual exploration + +MATCH (source_module:TS:Module)-[:DEPENDS_ON]->(target_module:TS:Module) +MATCH (source_project:TS:Project)-[:CONTAINS]->(source_module) +MATCH (target_project:TS:Project)-[:CONTAINS]->(target_module) +OPTIONAL MATCH (source_project)<-[:REFERENCED_PROJECTS*1..3]-(source_project_roots:TS:Project) +MATCH (source_scan:TS:Scan)-[:CONTAINS_PROJECT]->(source_project) +MATCH (target_scan:TS:Scan)-[:CONTAINS_PROJECT]->(target_project) +RETURN source_scan, target_scan + ,collect(source_project)[0..2], collect(target_project)[0..2] + ,collect(source_module)[0..9], collect(target_module)[0..9] +LIMIT 50 \ No newline at end of file diff --git a/cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationship_for_matching_modules.cypher b/cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationship_for_matching_modules.cypher index d2cb79bdd..b03658972 100644 --- a/cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationship_for_matching_modules.cypher +++ b/cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationship_for_matching_modules.cypher @@ -7,12 +7,13 @@ MATCH (module:TS:Module) WHERE module.globalFqn IS NOT NULL MATCH (externalModule:TS:ExternalModule) WHERE module.globalFqn IS NOT NULL - AND (module.globalFqn = externalModule.globalFqn - OR ( externalModule.name = module.name + AND ((module.globalFqn = externalModule.globalFqn) + OR (module.module = externalModule.module) + OR ( externalModule.name = module.name AND externalModule.moduleName = module.moduleName AND externalModule.namespace = module.namespace AND externalModule.extensionExtended = module.extensionExtended - ) + ) ) AND module <> externalModule CALL { WITH module, externalModule diff --git a/cypher/Typescript_Enrichment/Add_module_properties.cypher b/cypher/Typescript_Enrichment/Add_module_properties.cypher index 0949a733b..efac4a6da 100644 --- a/cypher/Typescript_Enrichment/Add_module_properties.cypher +++ b/cypher/Typescript_Enrichment/Add_module_properties.cypher @@ -18,7 +18,7 @@ OPTIONAL MATCH (class:TS:Class)-[:DECLARES]->(ts) ,coalesce('@' + nullif(namespaceName, ''), '') AS namespaceNameWithAtPrefixed ,replace(symbolName, coalesce(optionalClassName + '.', ''), '') AS symbolNameWithoutClassName SET ts.namespace = namespaceNameWithAtPrefixed - ,ts.module = modulePathName + ,ts.module = modulePathNameWithoutIndexAndDefault ,ts.moduleName = moduleName ,ts.name = coalesce(symbolNameWithoutClassName, indexAndExtensionOmittedName) ,ts.extensionExtended = moduleNameExtensionExtended diff --git a/cypher/Typescript_Enrichment/Count_internal_modules_resolving_external_ones.cypher b/cypher/Typescript_Enrichment/Count_internal_modules_resolving_external_ones.cypher new file mode 100644 index 000000000..dd448a1ee --- /dev/null +++ b/cypher/Typescript_Enrichment/Count_internal_modules_resolving_external_ones.cypher @@ -0,0 +1,14 @@ +// How many internal modules resolve/represent external ones (for manual exploration)? + + MATCH (internal_module:TS:Module) + WITH count(internal_module) AS totalNumberOfInternalModules + ,collect(internal_module) AS internal_modules + UNWIND internal_modules AS internal_module + MATCH (external_module:TS:ExternalModule)-[:RESOLVES_TO]->(internal_module) + WHERE NOT external_module.isNodeModule = true +RETURN totalNumberOfInternalModules + ,COUNT { (all_external_modules:TS:ExternalModule) + WHERE NOT all_external_modules.isNodeModule = true} + AS totalNumberOfExternalModules + ,count(DISTINCT external_module) AS numberOfResolvedExternalModules + ,collect(external_module.globalFqn + ' -> ' + internal_module.globalFqn)[0..4] AS exampleFullNames \ No newline at end of file diff --git a/cypher/Typescript_Enrichment/Link_external_modules_to_corresponding_npm_dependency.cypher b/cypher/Typescript_Enrichment/Link_external_modules_to_corresponding_npm_dependency.cypher new file mode 100644 index 000000000..1fa5a29c3 --- /dev/null +++ b/cypher/Typescript_Enrichment/Link_external_modules_to_corresponding_npm_dependency.cypher @@ -0,0 +1,20 @@ +// Link ExternalModule nodes to their corresponding NPM Dependency if available + + MATCH (external_module:TS:ExternalModule{isNodeModule:true}) + WITH count(external_module) AS totalNumberOfExternalModules + ,collect(external_module) AS external_modules +UNWIND external_modules AS external_module + MATCH (external_module)<-[:USES]-(project:TS:Project) + MATCH (project)-[:HAS_NPM_PACKAGE]->(npm_package:NPM:Package)-[npm_dependency_relation]->(npm_dependency:NPM:Dependency) + WHERE external_module.globalFqn CONTAINS ('/node_modules/' + npm_dependency.name) + CALL { WITH npm_dependency, external_module + MERGE (external_module)-[:PROVIDED_BY_NPM_DEPENDENCY]->(npm_dependency) + SET external_module.npmPackage = npm_dependency.name + ,external_module.npmPackageVersion = npm_dependency.dependency + } IN TRANSACTIONS +RETURN totalNumberOfExternalModules + ,count(DISTINCT external_module.globalFqn) AS numberOfLinkedExternalModules + ,count(DISTINCT project.name) AS numberOfProjects +//Debugging +//RETURN totalNumberOfExternalModules, project.name, npm_dependency.name, external_module.globalFqn +//LIMIT 40 \ No newline at end of file diff --git a/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher b/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher index 26bfc93a2..1c6ac30f0 100644 --- a/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher +++ b/cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher @@ -11,7 +11,7 @@ MATCH (npmPackage:NPM:Package) , '' ) AS relativeNpmPackageDirectory MATCH (project:TS:Project)-[:HAS_CONFIG]->(config:File)<-[:CONTAINS]-(projectConfigDir:Directory) - WHERE projectConfigDir.absoluteFileName ENDS WITH relativeNpmPackageDirectory + WHERE projectConfigDir.absoluteFileName ENDS WITH '/source' + relativeNpmPackageDirectory WITH npmPackage ,relativeNpmPackageDirectory ,collect(DISTINCT project) AS projects @@ -25,6 +25,7 @@ MATCH (npmPackage:NPM:Package) // Set the "relativeFileDirectory" on the npm package to the relative directory // that contains the package.json file SET npmPackage.relativeFileDirectory = relativeNpmPackageDirectory + ,project.version = npmPackage.version RETURN count(*) AS numberOfCreatedNpmPackageRelationships // Detailed results for debugging //RETURN npmPackage.fileName AS npmPackageFileName diff --git a/scripts/configuration/template-neo4jv5-jqassistant-continue-on-error.yaml b/scripts/configuration/template-neo4jv5-jqassistant-continue-on-error.yaml new file mode 100644 index 000000000..6b7151a4a --- /dev/null +++ b/scripts/configuration/template-neo4jv5-jqassistant-continue-on-error.yaml @@ -0,0 +1,228 @@ +jqassistant: + # Controls whether the execution of jQAssistant shall be skipped + # + # -Djqassistant.skip: true|false + skip: false + + # Defines the local and remote Maven repositories for retrieving additional plugins. + repositories: + # The path to the local repository. + # + # -Djqassistant.repositories.local (optional) + local: ./../downloads/jqassistant-repository + + # The list of plugins to load and activate. + # + # Each plugin is identified using its Maven coordinates: + # + # -Djqassistant.plugins[0].group-id + # -Djqassistant.plugins[0].artifact-id + # -Djqassistant.plugins[0].version + # -Djqassistant.plugins[0].classifier (optional) + # -Djqassistant.plugins[0].type (optional) + #plugins: + # - group-id: + # artifact-id: + # version: + # classifier: + # type: + plugins: + - group-id: org.jqassistant.plugin.typescript + artifact-id: jqassistant-typescript-plugin + version: 1.3.0 + - group-id: org.jqassistant.plugin + artifact-id: jqassistant-npm-plugin + version: 2.0.0 + - group-id: de.kontext-e.jqassistant.plugin + artifact-id: jqassistant.plugin.git + version: 2.1.0 + + + # The store configuration + store: + # URI of the database to connect to. Supported URI schemes are 'file' for embedded databases and 'bolt' for connecting to a running Neo4j instance (3.x+), e.g. + # + # -Djqassistant.store.uri + #uri: file://target/jqassistant/store + uri: bolt://localhost:${NEO4J_BOLT_PORT:7687} + + # Settings for the embedded Neo4j store + embedded: + # Enable the HTTP and BOLT connector for the embedded store + # + # -Djqassistant.store.embedded.connector-enabled: true|false + connector-enabled: false + + # The listen address for the HTTP and BOLT connectors + # + # -Djqassistant.store.embedded.listen-address + listen-address: localhost + + # The BOLT connector port + # + # -Djqassistant.store.embedded.bolt-port + bolt-port: 7687 + + # The HTTP connector port + # + # -Djqassistant.store.embedded.http-port: 7474 + http-port: 7474 + + # Settings for connecting to a remote Neo4j store + remote: + # The user name for authentication. + # + # -Djqassistant.store.username + username: ${NEO4J_USER:neo4j} + + # The password for authentication. + # + # -Djqassistant.store.password + password: ${NEO4J_INITIAL_PASSWORD} + + # Activate encryption level for 'bolt' connections. + # + # -Djqassistant.store.encryption: true|false + encryption: false + + #The trust strategy for 'bolt' connections + # + # -Djqassistant.store.trust-strategy: trustAllCertificates|trustCustomCaSignedCertificates|trustSystemCaSignedCertificates + trust-strategy: trustAllCertificates + + # The file containing the custom CA certificate for trust strategy. + # + # -Djqassistant.store.trust-certificate + trust-certificate: + + # Additional properties to be passed to the remote store as key-value pairs. + # + # -Djqassistant.store.properties + properties: + + # The Scanner configuration + scan: + # Indicates whether to initially reset the store (i.e. clear all nodes and relationships) before scanning. + # + # -Djqassistant.scan.reset: true|false + reset: true + + # Specifies if a scan shall be continued if an error is encountered. + # + # -Djqassistant.scan.continue-on-error: true|false + continue-on-error: true + + # The items to include for scanning. + include: + # A list of files to include. + # + #jqassistant.scan.include.files[0] + files: + # - src/folder + + # A list of URLs to include. + # + #jqassistant.scan.include.urls[0] + urls: + # - maven:repository::https://nexus/repository + + # The properties to configure scanner plugins as key-value pairs. The supported properties are plugin specific. + # + # -Djqassistant.scan.properties + properties: + # plugin.property.key: value + file.exclude: "*/.DS_Store" + json.file.exclude: "*/.reports/jqa/ts-output.json" + + # The analyze configuration + analyze: + # The rule configuration + rule: + # The name of the directory containing project rules. + # + # -Djqassistant.analyze.rule.directory + directory: jqassistant/rules + + # The default severity of concepts without an explicit severity. + # + # -Djqassistant.analyze.rule.default-concept-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER + default-concept-severity: MINOR + + # The default severity of constraint without an explicit severity. + # + # -Djqassistant.analyze.rule.default-constraint-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER + default-constraint-severity: MAJOR + + # The default severity of groups without an explicit severity. + # + # -Djqassistant.analyze.rule.default-group-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER + default-group-severity: + + # The report configuration + report: + # The properties to configure report plugins. The supported properties are plugin specific. + # + # -Djqassistant.analyze.report.properties + properties: + # plugin.property.key: value + + # Determines the severity level for reporting a warning. + # + # -Djqassistant.analyze.report.warn-on-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER|NEVER + warn-on-severity: MINOR + + # Determines the severity level for reporting a failure. + # + # -Djqassistant.analyze.report.fail-on-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER|NEVER + fail-on-severity: MAJOR + + # Determines if jQAssistant shall continue the build if failures have been detected. + # + # -Djqassistant.analyze.report.continue-on-failure: true|false + continue-on-failure: false + + # Create an archive containing all generated reports. + # + # -Djqassistant.analyze.report.create-archive: true|false + create-archive: false + + # Output directory for analysis reports + directory: ./runtime/jqassistant/report + + # The concepts to be applied. + # + # -Djqassistant.analyze.concepts[0] + # https://github.com/jQAssistant/jqa-java-plugin/blob/e17fc3032fb0d4258d4f5ca0c64099a6c789c070/src/main/asciidoc/release-notes.adoc#L9 + concepts: + - java-classpath:Resolve + - java:PackageDependency + - java:ArtifactDependency + - java:JavaVersion + + # The constraints to be validated. + # + # -Djqassistant.analyze.constraints[0] + constraints: + # - my-constraint + + # The groups to be executed. + # + # -Djqassistant.analyze.groups[0] + groups: + # - spring-boot:Default + + # The parameters to be passed to the executed rules. + # + # -Djqassistant.analyze.rule-parameters."parameterName" + rule-parameters: + # parameterName: value + + # Execute concepts even if they have already been applied before + # + # -Djqassistant.analyze.execute-applied-concepts: true|false + execute-applied-concepts: false + + # The execution time [seconds] for rules (concepts/constraints) to show a warning. Can be used as a hint for optimization. + # + # -Djqassistant.analyze.warn-on-rule-execution-time-seconds + warn-on-rule-execution-time-seconds: 5 diff --git a/scripts/findPathsToScan.sh b/scripts/findPathsToScan.sh index 6ff734b1e..ea60142e1 100755 --- a/scripts/findPathsToScan.sh +++ b/scripts/findPathsToScan.sh @@ -44,6 +44,7 @@ if [ -d "./${SOURCE_DIRECTORY}" ] ; then -type d -name "node_modules" -prune -o \ -type d -name "dist" -prune -o \ -type d -name "target" -prune -o \ + -type d -name ".yalc" -prune -o \ -type d -name "temp" -prune -o \ -type f -path "*/.reports/jqa/ts-output.json" \ -exec echo typescript:project::{} \; | tr '\n' ',' | sed 's/,$/\n/')" diff --git a/scripts/prepareAnalysis.sh b/scripts/prepareAnalysis.sh index 058ea7c19..8d5a1d450 100644 --- a/scripts/prepareAnalysis.sh +++ b/scripts/prepareAnalysis.sh @@ -87,6 +87,7 @@ if is_csv_column_greater_zero "${dataVerificationResult}" "unresolvedProjectsCou echo -e "${COLOR_RED}${dataVerificationResult}${COLOR_DEFAULT}" exit 1 fi +execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_external_modules_to_corresponding_npm_dependency.cypher" # Preparation - Add weights to Java Package DEPENDS_ON relationships execute_cypher_summarized "${DEPENDS_ON_CYPHER_DIR}/Add_weight_property_for_Java_Interface_Dependencies_to_Package_DEPENDS_ON_Relationship.cypher" @@ -117,9 +118,10 @@ execute_cypher "${TYPES_CYPHER_DIR}/Label_resolved_duplicate_types.cypher" execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/Remove_external_type_and_annotation_labels.cypher" execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/Label_external_types_and_annotations.cypher" -# Preparation - Add Java Artifact node properties "incomingDependencies" and "outgoingDependencies" +# Preparation - Add Java Artifact node properties "incomingDependencies", "outgoingDependencies" and "version" execute_cypher_summarized "${ARTIFACT_DEPENDENCIES_CYPHER_DIR}/Incoming_Java_Artifact_Dependencies.cypher" execute_cypher_summarized "${ARTIFACT_DEPENDENCIES_CYPHER_DIR}/Outgoing_Java_Artifact_Dependencies.cypher" +execute_cypher_summarized "${ARTIFACT_DEPENDENCIES_CYPHER_DIR}/Set_maven_artifact_version.cypher" # Preparation - Add Java Type node properties "incomingDependencies" and "outgoingDependencies" execute_cypher_summarized "${METRICS_CYPHER_DIR}/Set_Incoming_Java_Type_Dependencies.cypher" diff --git a/scripts/profiles/Neo4jv4.sh b/scripts/profiles/Neo4jv4.sh index 141bfd12c..6b31b8a6a 100755 --- a/scripts/profiles/Neo4jv4.sh +++ b/scripts/profiles/Neo4jv4.sh @@ -19,8 +19,8 @@ NEO4J_APOC_PLUGIN_GITHUB=${NEO4J_APOC_PLUGIN_GITHUB:-"neo4j-contrib/neo4j-apoc-p NEO4J_GDS_PLUGIN_VERSION=${NEO4J_GDS_PLUGIN_VERSION:-"2.3.4"} # Graph Data Science Plugin Version 2.3.x is compatible with Neo4j 4.4.x NEO4J_OPEN_GDS_PLUGIN_VERSION=${NEO4J_OPEN_GDS_PLUGIN_VERSION:-"2.6.8"} # Open package variant of the graph-data-science plugin for Neo4j (https://github.com/JohT/open-graph-data-science-packaging). Since version 2.4. compatible with Neo4j 5.x. -NEO4J_GDS_PLUGIN_EDITION=${NEO4J_GDS_PLUGIN_EDITION:-"open"} # Graph Data Science Plugin Edition: "open" for OpenGDS, "full" for the full version with Neo4j license +NEO4J_GDS_PLUGIN_EDITION=${NEO4J_GDS_PLUGIN_EDITION:-"open"} # Graph Data Science Plugin Edition: "open" for OpenGDS, "full" for the full version with Neo4j license -JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"1.12.2"} # Version 1.12.2 is the newest version (may 2023) compatible with Neo4j v4 +JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"1.12.2"} # Version number of the jQAssistant command line interface. Version 1.12.2 is compatible with Neo4j v4 JQASSISTANT_CLI_ARTIFACT=${JQASSISTANT_CLI_ARTIFACT:-"jqassistant-commandline-neo4jv4"} # For Neo4j 4: "jqassistant-commandline-neo4jv4" JQASSISTANT_CONFIG_TEMPLATE=${JQASSISTANT_CONFIG_TEMPLATE:-"template-neo4jv4-jqassistant.yaml"} # Name of the template file for the jqassistant configuration \ No newline at end of file diff --git a/scripts/profiles/Neo4jv5-continue-on-scan-errors.sh b/scripts/profiles/Neo4jv5-continue-on-scan-errors.sh new file mode 100755 index 000000000..fbebcd12b --- /dev/null +++ b/scripts/profiles/Neo4jv5-continue-on-scan-errors.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Sets all settings variables for an analysis with Neo4j v5.x (newest version as of june 2023). +# The chosen settings are tested to be compatible and working. + +NEO4J_VERSION=${NEO4J_VERSION:-"5.23.0"} # Version 5.9.0 is the current version of june 2023 +NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/neo4j/tx/commit"} # Since Neo4j v5 it is "db//tx/commit" + +# Overrideable settings variables for ports (optional, defaults also defined in sub scripts where needed) +# Override them if you need to run multiple neo4j database servers in parallel. +NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} # Neo4j HTTP API port for executing queries +NEO4J_HTTPS_PORT=${NEO4J_HTTPS_PORT:-"7473"} # Neo4j HTTPS port for encrypted querying +NEO4J_BOLT_PORT=${NEO4J_BOLT_PORT:-"7687"} # Neo4j's own "Bolt Protocol" port + +# Awesome Procedures (APOC) Plugin for Neo4j +NEO4J_APOC_PLUGIN_VERSION=${NEO4J_APOC_PLUGIN_VERSION:-"5.24.0"} # Version number matches Neo4j version since 5.x +NEO4J_APOC_PLUGIN_EDITION=${NEO4J_APOC_PLUGIN_EDITION:-"core"} # Since Neo4j v5 the core edition is updated with Neo4j +NEO4J_APOC_PLUGIN_GITHUB=${NEO4J_APOC_PLUGIN_GITHUB:-"neo4j/apoc"} # Core edition was moved to "neo4j/apoc" for Neo4j v5 + +NEO4J_GDS_PLUGIN_VERSION=${NEO4J_GDS_PLUGIN_VERSION:-"2.8.0"} # Version 2.4.0 is the newest version of june 2023 and compatible with Neo4j v5 +NEO4J_OPEN_GDS_PLUGIN_VERSION=${NEO4J_OPEN_GDS_PLUGIN_VERSION:-"2.11.0"} # Open package variant of the graph-data-science plugin for Neo4j (https://github.com/JohT/open-graph-data-science-packaging). Since version 2.4. compatible with Neo4j 5.x. +NEO4J_GDS_PLUGIN_EDITION=${NEO4J_GDS_PLUGIN_EDITION:-"open"} # Graph Data Science Plugin Edition: "open" for OpenGDS, "full" for the full version with Neo4j license + +JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.3.1"} # Version number of the jQAssistant command line interface. Version 1.12.2 is compatible with Neo4j v4 +JQASSISTANT_CLI_ARTIFACT=${JQASSISTANT_CLI_ARTIFACT:-"template-neo4jv5-jqassistant-continue-on-error"} +JQASSISTANT_CONFIG_TEMPLATE=${JQASSISTANT_CONFIG_TEMPLATE:-"template-neo4jv5-jqassistant.yaml"} # Name of the template file for the jqassistant configuration \ No newline at end of file diff --git a/scripts/profiles/Neo4jv5.sh b/scripts/profiles/Neo4jv5.sh index d509864f0..c4101732a 100755 --- a/scripts/profiles/Neo4jv5.sh +++ b/scripts/profiles/Neo4jv5.sh @@ -21,6 +21,6 @@ NEO4J_GDS_PLUGIN_VERSION=${NEO4J_GDS_PLUGIN_VERSION:-"2.10.1"} # Version 2.4.0 NEO4J_OPEN_GDS_PLUGIN_VERSION=${NEO4J_OPEN_GDS_PLUGIN_VERSION:-"2.11.0"} # Open package variant of the graph-data-science plugin for Neo4j (https://github.com/JohT/open-graph-data-science-packaging). Since version 2.4. compatible with Neo4j 5.x. NEO4J_GDS_PLUGIN_EDITION=${NEO4J_GDS_PLUGIN_EDITION:-"open"} # Graph Data Science Plugin Edition: "open" for OpenGDS, "full" for the full version with Neo4j license -JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.3.1"} # Version 2.0.3 is the newest version (june 2023) compatible with Neo4j v5 +JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.3.1"} # Version number of the jQAssistant command line interface. Version 1.12.2 is compatible with Neo4j v4 JQASSISTANT_CLI_ARTIFACT=${JQASSISTANT_CLI_ARTIFACT:-"jqassistant-commandline-neo4jv5"} # For Neo4jv5: "jqassistant-commandline-distribution" JQASSISTANT_CONFIG_TEMPLATE=${JQASSISTANT_CONFIG_TEMPLATE:-"template-neo4jv5-jqassistant.yaml"} # Name of the template file for the jqassistant configuration \ No newline at end of file diff --git a/scripts/resetAndScan.sh b/scripts/resetAndScan.sh index 9ecfa902b..8b695ad94 100755 --- a/scripts/resetAndScan.sh +++ b/scripts/resetAndScan.sh @@ -13,7 +13,7 @@ # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) set -o errexit -o pipefail -JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.3.1"} # 2.0.3 is the newest version (june 2023) compatible with Neo4j v5, Version 1.12.2 is compatible with Neo4j v4 +JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.3.1"} # Version number of the jQAssistant command line interface. Version 1.12.2 is compatible with Neo4j v4 JQASSISTANT_CLI_ARTIFACT=${JQASSISTANT_CLI_ARTIFACT:-"jqassistant-commandline-neo4jv5"} # Neo4j v5: "jqassistant-commandline-neo4jv5", Neo4j v4: "jqassistant-commandline-neo4jv4" JQASSISTANT_CONFIG_TEMPLATE=${JQASSISTANT_CONFIG_TEMPLATE:-"template-neo4jv5-jqassistant.yaml"} # Neo4j v5: "template-neo4jv5-jqassistant.yaml", Neo4j v4: "template-neo4jv4-jqassistant.yaml" diff --git a/scripts/scanTypescript.sh b/scripts/scanTypescript.sh index beb75af38..916170d1a 100755 --- a/scripts/scanTypescript.sh +++ b/scripts/scanTypescript.sh @@ -4,19 +4,30 @@ # Uses: patchJQAssistantTypescriptPlugin.sh, detectChangedFiles.sh +# Note: This script must not output anything except for the return code to stdout. +# All output of the scanning needs to be redirected to stderr using ">&2". + # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) set -o errexit -o pipefail ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"} SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} + TYPESCRIPT_SCAN_HEAP_MEMORY=${TYPESCRIPT_SCAN_HEAP_MEMORY:-"4096"} # Heap memory in megabytes for Typescript scanning with (Node.js process). Defaults to 4096 MB. +echo "scanTypescript: TYPESCRIPT_SCAN_HEAP_MEMORY=${TYPESCRIPT_SCAN_HEAP_MEMORY}" >&2 ## Get this "scripts" directory if not already set # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. # This way non-standard tools like readlink aren't needed. SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts -echo "scanTypescript SCRIPTS_DIR=${SCRIPTS_DIR}" >&2 +echo "scanTypescript: SCRIPTS_DIR=${SCRIPTS_DIR}" >&2 + +# Dry run for internal testing (not intended to be accessible from the outside) +TYPESCRIPT_SCAN_DRY_RUN=false +if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = true ] ; then + echo "scanTypescript: -> DRY RUN <- Scanning will only be logged, not executed." >&2 +fi if [ ! -d "./${SOURCE_DIRECTORY}" ] ; then echo "scanTypescript: Source directory '${SOURCE_DIRECTORY}' doesn't exist. The scan will therefore be skipped." >&2 @@ -32,9 +43,11 @@ fi changeDetectionHashFilePath="./${SOURCE_DIRECTORY}/typescriptFileChangeDetectionHashFile.txt" changeDetectionReturnCode=$( source "${SCRIPTS_DIR}/detectChangedFiles.sh" --readonly --hashfile "${changeDetectionHashFilePath}" --paths "./${SOURCE_DIRECTORY}") -if [ "${changeDetectionReturnCode}" == "0" ] ; then +if [ "${changeDetectionReturnCode}" == "0" ]; then echo "scanTypescript: Files unchanged. Scan skipped." -else +fi + +if [ "${changeDetectionReturnCode}" != "0" ] || [ "${TYPESCRIPT_SCAN_DRY_RUN}" = true ]; then echo "scanTypescript: Detected change (${changeDetectionReturnCode}). Scanning Typescript source using @jqassistant/ts-lce." mkdir -p "./runtime/logs" @@ -42,6 +55,7 @@ else echo "scanTypescript: LOG_DIRECTORY=${LOG_DIRECTORY}" >&2 source_directories=$( find -L "./${SOURCE_DIRECTORY}" -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -r -I {} echo {} ) + total_directories=$(echo "${source_directories}" | wc -l | awk '{print $1}') processed_directories=0 @@ -50,13 +64,16 @@ else processed_directories=$((processed_directories + 1)) echo "" >&2 echo "scanTypescript: $(date +'%Y-%m-%dT%H:%M:%S%z') Scanning ${directory_name} (${processed_directories}/${total_directories}) -----------------" >&2 - # Note: This script must not output anything except for the return code to stdout, - # all output of the scanning needs to be redirected to stderr using ">&2". - # For later troubleshooting, the output is also copied to a dedicated log file using "tee". - # Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager. - # Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for larger projects to scan - NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=${TYPESCRIPT_SCAN_HEAP_MEMORY}" npx --yes @jqassistant/ts-lce@1.3.0 "${directory}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2 + + if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = false ] ; then + # Note: For later troubleshooting, the output is also copied to a dedicated log file using "tee". + # Note: Don't worry about the hardcoded version number. It will be updated by Renovate using a custom Manager. + # Note: NODE_OPTIONS --max-old-space-size=4096 increases the memory for scanning larger projects + NODE_OPTIONS="${NODE_OPTIONS} --max-old-space-size=${TYPESCRIPT_SCAN_HEAP_MEMORY}" npx --yes @jqassistant/ts-lce@1.3.0 "${directory}" --extension React 2>&1 | tee "${LOG_DIRECTORY}/jqassistant-typescript-scan-${directory_name}.log" >&2 + fi done - changeDetectionReturnCode=$( source "${SCRIPTS_DIR}/detectChangedFiles.sh" --hashfile "${changeDetectionHashFilePath}" --paths "./${SOURCE_DIRECTORY}") + if [ "${TYPESCRIPT_SCAN_DRY_RUN}" = false ] ; then + changeDetectionReturnCode=$( source "${SCRIPTS_DIR}/detectChangedFiles.sh" --hashfile "${changeDetectionHashFilePath}" --paths "./${SOURCE_DIRECTORY}") + fi fi \ No newline at end of file diff --git a/scripts/setupJQAssistant.sh b/scripts/setupJQAssistant.sh index 7f4acfa42..9d8661ed1 100755 --- a/scripts/setupJQAssistant.sh +++ b/scripts/setupJQAssistant.sh @@ -10,7 +10,7 @@ # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) set -o errexit -o pipefail -JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.3.1"} # Neo4j v5: 2.0.3 (june 2023), Neo4j v4: 1.12.2 (april 2023) +JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.3.1"} # Version number of the jQAssistant command line interface. Version 1.12.2 is compatible with Neo4j v4 JQASSISTANT_CLI_DOWNLOAD_URL=${JQASSISTANT_CLI_DOWNLOAD_URL:-"https://repo1.maven.org/maven2/com/buschmais/jqassistant/cli"} # Download URL for the jQAssistant CLI JQASSISTANT_CLI_ARTIFACT=${JQASSISTANT_CLI_ARTIFACT:-"jqassistant-commandline-neo4jv5"} # Neo4j v5: "jqassistant-commandline-neo4jv5", Neo4j v4: "jqassistant-commandline-neo4jv4" JQASSISTANT_CLI_DISTRIBUTION=${JQASSISTANT_CLI_DISTRIBUTION:-"distribution.zip"} # Neo4j v5 & v4: "distribution.zip"