Support Typescript version numbers

COMMANDS.md

@@ -71,7 +71,9 @@ The [analyze.sh](./scripts/analysis/analyze.sh) command comes with these command
 - `--report Csv` only generates CSV reports. This speeds up the report generation and doesn't depend on Python, Jupyter Notebook or any other related dependencies. The default value os `All` to generate all reports. `Jupiter` will only generate Jupyter Notebook reports. `DatabaseCsvExport` exports the whole graph database as a CSV file (performance intense, check if there are security concerns first).
 
 - `--profile Neo4jv4` uses the older long term support (june 2023) version v4.4.x of Neo4j and suitable compatible versions of plugins and JQAssistant. `Neo4jv5` will explicitly select the newest (june 2023) version 5.x of Neo4j. Without setting
-a profile, the newest versions will be used. Profiles are scripts that can be found in the directory [scripts/profiles](./scripts/profiles/).
+a profile, the newest versions will be used. Profiles can be found in the directory [scripts/profiles](./scripts/profiles/).
+
+- `--profile Neo4jv5-continue-on-scan-errors` is based on the default profile (`Neo4jv5`) but uses the jQAssistant configuration template [template-neo4jv5-jqassistant-continue-on-error.yaml](./scripts/configuration/template-neo4jv5-jqassistant-continue-on-error.yaml) to continue on scan error instead of failing fast. This is temporarily useful when there is a known error that needs to be ignored. It is still recommended to use the default profile and fail fast if there is something wrong. Profiles can be found in the directory [scripts/profiles](./scripts/profiles/).
 
 - `--explore` activates the "explore" mode where no reports are generated. Furthermore, Neo4j won't be stopped at the end of the script and will therefore continue running.  This makes it easy to just set everything up but then use the running Neo4j server to explore the data manually.
 

README.md

@@ -226,6 +226,14 @@ The [Code Structure Analysis Pipeline](./.github/workflows/java-code-analysis.ym
   TYPESCRIPT_SCAN_HEAP_MEMORY=16384 ./../../scripts/analysis/analyze.sh
   ```
 
+- How can i continue on errors when scanning Typescript projects instead of cancelling the whole analysis?  
+  👉 Use the profile `Neo4jv5-continue-on-scan-errors` (default = `Neo4jv5`):
+
+  ```shell
+  ./../../scripts/analysis/analyze.sh --profile Neo4jv5-continue-on-scan-errors
+  ```
+
+
 ## 🕸 Web References
 
 - [Graph Data Science 101: Understanding Graphs and Graph Data Science](https://techfirst.medium.com/graph-data-science-101-understanding-graphs-and-graph-data-science-c25055a9db01)

cypher/Artifact_Dependencies/Set_maven_artifact_version.cypher

@@ -0,0 +1,14 @@
+// Set property "version" on Artifact nodes to what is specified in the Maven POM
+
+   MATCH (artifact:Java:Artifact)-[:CONTAINS]-(pom:Maven:Pom)
+   MATCH (pom)-[:HAS_ROOT_ELEMENT]->(pomProject:Element{name:'project'})
+OPTIONAL MATCH (pomProject)-[:HAS_ELEMENT]->(:Element{name:'version'})-[:HAS_TEXT]->(versionValue:Text) 
+OPTIONAL MATCH (pomProject)-[:HAS_ELEMENT]->(:Element{name:'parent'})
+-[:HAS_ELEMENT]->(:Element{name:'version'})-[:HAS_TEXT]->(parentVersionValue:Text)
+   WITH *, artifact, coalesce(versionValue.value, parentVersionValue.value) AS versionNumber
+  WHERE versionNumber IS NOT NULL
+    SET artifact.version = versionNumber
+  RETURN count(DISTINCT artifact.fileName) AS numberOfUpdatedArtifacts
+// Debugging
+// RETURN artifact.name, versionNumber, versionValue.value, parentVersionValue.value
+//  LIMIT 25

cypher/Internal_Dependencies/Inter_scan_and_project_dependencies_of_Typescript_modules.cypher

@@ -0,0 +1,12 @@
+// Inter scan/project higher-level module dependencies for manual exploration
+
+MATCH (source_module:TS:Module)-[:DEPENDS_ON]->(target_module:TS:Module)
+MATCH (source_project:TS:Project)-[:CONTAINS]->(source_module)
+MATCH (target_project:TS:Project)-[:CONTAINS]->(target_module)
+OPTIONAL MATCH (source_project)<-[:REFERENCED_PROJECTS*1..3]-(source_project_roots:TS:Project)
+MATCH (source_scan:TS:Scan)-[:CONTAINS_PROJECT]->(source_project)
+MATCH (target_scan:TS:Scan)-[:CONTAINS_PROJECT]->(target_project)
+RETURN source_scan, target_scan
+      ,collect(source_project)[0..2], collect(target_project)[0..2]
+      ,collect(source_module)[0..9], collect(target_module)[0..9]
+LIMIT 50

cypher/Typescript_Enrichment/Add_RESOLVES_TO_relationship_for_matching_modules.cypher

@@ -7,12 +7,13 @@ MATCH (module:TS:Module)
 WHERE  module.globalFqn IS NOT NULL
 MATCH (externalModule:TS:ExternalModule)
 WHERE module.globalFqn IS NOT NULL
-  AND (module.globalFqn = externalModule.globalFqn
-   OR   ( externalModule.name              = module.name 
+  AND ((module.globalFqn = externalModule.globalFqn)
+   OR  (module.module    = externalModule.module)
+   OR  (  externalModule.name              = module.name 
       AND externalModule.moduleName        = module.moduleName 
       AND externalModule.namespace         = module.namespace
       AND externalModule.extensionExtended = module.extensionExtended
-        )
+       )
       )
   AND module <> externalModule
  CALL {  WITH module, externalModule

cypher/Typescript_Enrichment/Add_module_properties.cypher

@@ -18,7 +18,7 @@ OPTIONAL MATCH (class:TS:Class)-[:DECLARES]->(ts)
       ,coalesce('@' + nullif(namespaceName, ''), '')                                        AS namespaceNameWithAtPrefixed
       ,replace(symbolName, coalesce(optionalClassName + '.', ''), '')                       AS symbolNameWithoutClassName
     SET ts.namespace          = namespaceNameWithAtPrefixed
-       ,ts.module             = modulePathName
+       ,ts.module             = modulePathNameWithoutIndexAndDefault
        ,ts.moduleName         = moduleName
        ,ts.name               = coalesce(symbolNameWithoutClassName, indexAndExtensionOmittedName)
        ,ts.extensionExtended  = moduleNameExtensionExtended

cypher/Typescript_Enrichment/Count_internal_modules_resolving_external_ones.cypher

@@ -0,0 +1,14 @@
+// How many internal modules resolve/represent external ones (for manual exploration)?
+
+ MATCH (internal_module:TS:Module)
+  WITH count(internal_module)   AS totalNumberOfInternalModules
+      ,collect(internal_module) AS internal_modules
+ UNWIND internal_modules AS internal_module
+ MATCH (external_module:TS:ExternalModule)-[:RESOLVES_TO]->(internal_module)
+ WHERE NOT external_module.isNodeModule = true
+RETURN totalNumberOfInternalModules
+      ,COUNT { (all_external_modules:TS:ExternalModule) 
+               WHERE NOT all_external_modules.isNodeModule = true} 
+                                       AS totalNumberOfExternalModules
+      ,count(DISTINCT external_module) AS numberOfResolvedExternalModules
+      ,collect(external_module.globalFqn + ' -> ' + internal_module.globalFqn)[0..4] AS exampleFullNames

cypher/Typescript_Enrichment/Link_external_modules_to_corresponding_npm_dependency.cypher

@@ -0,0 +1,20 @@
+// Link ExternalModule nodes to their corresponding NPM Dependency if available
+
+ MATCH (external_module:TS:ExternalModule{isNodeModule:true})
+  WITH count(external_module)   AS totalNumberOfExternalModules
+      ,collect(external_module) AS external_modules
+UNWIND external_modules         AS external_module 
+ MATCH (external_module)<-[:USES]-(project:TS:Project)
+ MATCH (project)-[:HAS_NPM_PACKAGE]->(npm_package:NPM:Package)-[npm_dependency_relation]->(npm_dependency:NPM:Dependency)
+ WHERE external_module.globalFqn CONTAINS ('/node_modules/' + npm_dependency.name)
+ CALL {  WITH npm_dependency, external_module
+        MERGE (external_module)-[:PROVIDED_BY_NPM_DEPENDENCY]->(npm_dependency)
+          SET external_module.npmPackage        = npm_dependency.name
+             ,external_module.npmPackageVersion = npm_dependency.dependency
+      } IN TRANSACTIONS
+RETURN totalNumberOfExternalModules
+      ,count(DISTINCT external_module.globalFqn) AS numberOfLinkedExternalModules
+      ,count(DISTINCT project.name)              AS numberOfProjects
+//Debugging
+//RETURN totalNumberOfExternalModules, project.name, npm_dependency.name, external_module.globalFqn
+//LIMIT 40

cypher/Typescript_Enrichment/Link_projects_to_npm_packages.cypher

@@ -11,7 +11,7 @@ MATCH (npmPackage:NPM:Package)
           , ''
           ) AS relativeNpmPackageDirectory
  MATCH (project:TS:Project)-[:HAS_CONFIG]->(config:File)<-[:CONTAINS]-(projectConfigDir:Directory)
- WHERE projectConfigDir.absoluteFileName ENDS WITH relativeNpmPackageDirectory
+ WHERE projectConfigDir.absoluteFileName ENDS WITH '/source' + relativeNpmPackageDirectory
   WITH npmPackage
       ,relativeNpmPackageDirectory
       ,collect(DISTINCT project) AS projects
@@ -25,6 +25,7 @@ MATCH (npmPackage:NPM:Package)
 // Set the "relativeFileDirectory" on the npm package to the relative directory 
 // that contains the package.json file 
    SET npmPackage.relativeFileDirectory = relativeNpmPackageDirectory
+      ,project.version = npmPackage.version
  RETURN count(*) AS numberOfCreatedNpmPackageRelationships
 // Detailed results for debugging
 //RETURN npmPackage.fileName                   AS npmPackageFileName

scripts/configuration/template-neo4jv5-jqassistant-continue-on-error.yaml

@@ -0,0 +1,228 @@
+jqassistant:
+  # Controls whether the execution of jQAssistant shall be skipped
+  #
+  # -Djqassistant.skip: true|false
+  skip: false
+
+  # Defines the local and remote Maven repositories for retrieving additional plugins.
+  repositories:
+    # The path to the local repository.
+    #
+    # -Djqassistant.repositories.local (optional)
+    local: ./../downloads/jqassistant-repository
+
+  # The list of plugins to load and activate.
+  #
+  # Each plugin is identified using its Maven coordinates:
+  #
+  # -Djqassistant.plugins[0].group-id
+  # -Djqassistant.plugins[0].artifact-id
+  # -Djqassistant.plugins[0].version
+  # -Djqassistant.plugins[0].classifier (optional)
+  # -Djqassistant.plugins[0].type (optional)
+  #plugins:
+  #  - group-id:
+  #    artifact-id:
+  #    version:
+  #    classifier:
+  #    type:
+  plugins:
+    - group-id: org.jqassistant.plugin.typescript
+      artifact-id: jqassistant-typescript-plugin
+      version: 1.3.0
+    - group-id: org.jqassistant.plugin
+      artifact-id: jqassistant-npm-plugin
+      version: 2.0.0
+    - group-id: de.kontext-e.jqassistant.plugin
+      artifact-id: jqassistant.plugin.git
+      version: 2.1.0
+
+
+  # The store configuration
+  store:
+    # URI of the database to connect to. Supported URI schemes are 'file' for embedded databases and 'bolt' for connecting to a running Neo4j instance (3.x+), e.g.
+    #
+    # -Djqassistant.store.uri
+    #uri: file://target/jqassistant/store
+    uri: bolt://localhost:${NEO4J_BOLT_PORT:7687}
+
+    # Settings for the embedded Neo4j store
+    embedded:
+      # Enable the HTTP and BOLT connector for the embedded store
+      #
+      # -Djqassistant.store.embedded.connector-enabled: true|false
+      connector-enabled: false
+
+      # The listen address for the HTTP and BOLT connectors
+      #
+      # -Djqassistant.store.embedded.listen-address
+      listen-address: localhost
+
+      # The BOLT connector port
+      #
+      # -Djqassistant.store.embedded.bolt-port
+      bolt-port: 7687
+
+      # The HTTP connector port
+      #
+      # -Djqassistant.store.embedded.http-port: 7474
+      http-port: 7474
+
+    # Settings for connecting to a remote Neo4j store
+    remote:
+      # The user name for authentication.
+      #
+      # -Djqassistant.store.username
+      username: ${NEO4J_USER:neo4j}
+
+      # The password for authentication.
+      #
+      # -Djqassistant.store.password
+      password: ${NEO4J_INITIAL_PASSWORD}
+
+      # Activate encryption level for 'bolt' connections.
+      #
+      # -Djqassistant.store.encryption: true|false
+      encryption: false
+
+      #The trust strategy for 'bolt' connections
+      #
+      # -Djqassistant.store.trust-strategy: trustAllCertificates|trustCustomCaSignedCertificates|trustSystemCaSignedCertificates
+      trust-strategy: trustAllCertificates
+
+      # The file containing the custom CA certificate for trust strategy.
+      #
+      # -Djqassistant.store.trust-certificate
+      trust-certificate:
+
+      # Additional properties to be passed to the remote store as key-value pairs.
+      #
+      # -Djqassistant.store.properties
+      properties:
+
+  # The Scanner configuration
+  scan:
+    # Indicates whether to initially reset the store (i.e. clear all nodes and relationships) before scanning.
+    #
+    # -Djqassistant.scan.reset: true|false
+    reset: true
+
+    # Specifies if a scan shall be continued if an error is encountered.
+    #
+    # -Djqassistant.scan.continue-on-error: true|false
+    continue-on-error: true
+
+    # The items to include for scanning.
+    include:
+      # A list of files to include.
+      #
+      #jqassistant.scan.include.files[0]
+      files:
+      # - src/folder
+
+      # A list of URLs to include.
+      #
+      #jqassistant.scan.include.urls[0]
+      urls:
+      # - maven:repository::https://nexus/repository
+
+    # The properties to configure scanner plugins as key-value pairs. The supported properties are plugin specific.
+    #
+    # -Djqassistant.scan.properties
+    properties:
+      # plugin.property.key: value
+      file.exclude: "*/.DS_Store"
+      json.file.exclude: "*/.reports/jqa/ts-output.json"
+
+  # The analyze configuration
+  analyze:
+    # The rule configuration
+    rule:
+      # The name of the directory containing project rules.
+      #
+      # -Djqassistant.analyze.rule.directory
+      directory: jqassistant/rules
+
+      # The default severity of concepts without an explicit severity.
+      #
+      # -Djqassistant.analyze.rule.default-concept-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER
+      default-concept-severity: MINOR
+
+      # The default severity of constraint without an explicit severity.
+      #
+      # -Djqassistant.analyze.rule.default-constraint-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER
+      default-constraint-severity: MAJOR
+
+      # The default severity of groups without an explicit severity.
+      #
+      # -Djqassistant.analyze.rule.default-group-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER
+      default-group-severity:
+
+    # The report configuration
+    report:
+      # The properties to configure report plugins. The supported properties are plugin specific.
+      #
+      # -Djqassistant.analyze.report.properties
+      properties:
+        # plugin.property.key: value
+
+      # Determines the severity level for reporting a warning.
+      #
+      # -Djqassistant.analyze.report.warn-on-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER|NEVER
+      warn-on-severity: MINOR
+
+      # Determines the severity level for reporting a failure.
+      #
+      # -Djqassistant.analyze.report.fail-on-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER|NEVER
+      fail-on-severity: MAJOR
+
+      # Determines if jQAssistant shall continue the build if failures have been detected.
+      #
+      # -Djqassistant.analyze.report.continue-on-failure: true|false
+      continue-on-failure: false
+
+      # Create an archive containing all generated reports.
+      #
+      # -Djqassistant.analyze.report.create-archive: true|false
+      create-archive: false
+
+      # Output directory for analysis reports
+      directory: ./runtime/jqassistant/report
+
+    # The concepts to be applied.
+    #
+    # -Djqassistant.analyze.concepts[0]
+    # https://github.com/jQAssistant/jqa-java-plugin/blob/e17fc3032fb0d4258d4f5ca0c64099a6c789c070/src/main/asciidoc/release-notes.adoc#L9
+    concepts:
+     - java-classpath:Resolve
+     - java:PackageDependency
+     - java:ArtifactDependency
+     - java:JavaVersion
+
+    # The constraints to be validated.
+    #
+    # -Djqassistant.analyze.constraints[0]
+    constraints:
+    # - my-constraint
+
+    # The groups to be executed.
+    #
+    # -Djqassistant.analyze.groups[0]
+    groups:
+    # - spring-boot:Default
+
+    # The parameters to be passed to the executed rules.
+    #
+    # -Djqassistant.analyze.rule-parameters."parameterName"
+    rule-parameters:
+    # parameterName: value
+
+    # Execute concepts even if they have already been applied before
+    #
+    # -Djqassistant.analyze.execute-applied-concepts: true|false
+    execute-applied-concepts: false
+
+    # The execution time [seconds] for rules (concepts/constraints) to show a warning. Can be used as a hint for optimization.
+    #
+    # -Djqassistant.analyze.warn-on-rule-execution-time-seconds
+    warn-on-rule-execution-time-seconds: 5

scripts/findPathsToScan.sh

@@ -44,6 +44,7 @@ if [ -d "./${SOURCE_DIRECTORY}" ] ; then
                                     -type d -name "node_modules" -prune -o \
                                     -type d -name "dist" -prune -o \
                                     -type d -name "target" -prune -o \
+                                    -type d -name ".yalc" -prune -o \
                                     -type d -name "temp" -prune -o \
                                     -type f -path "*/.reports/jqa/ts-output.json" \
                                     -exec echo typescript:project::{} \; | tr '\n' ',' | sed 's/,$/\n/')"

scripts/prepareAnalysis.sh

@@ -87,6 +87,7 @@ if is_csv_column_greater_zero "${dataVerificationResult}" "unresolvedProjectsCou
     echo -e "${COLOR_RED}${dataVerificationResult}${COLOR_DEFAULT}"
     exit 1
 fi
+execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_external_modules_to_corresponding_npm_dependency.cypher"
 
 # Preparation - Add weights to Java Package DEPENDS_ON relationships 
 execute_cypher_summarized "${DEPENDS_ON_CYPHER_DIR}/Add_weight_property_for_Java_Interface_Dependencies_to_Package_DEPENDS_ON_Relationship.cypher"
@@ -117,9 +118,10 @@ execute_cypher "${TYPES_CYPHER_DIR}/Label_resolved_duplicate_types.cypher"
 execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/Remove_external_type_and_annotation_labels.cypher"
 execute_cypher "${EXTERNAL_DEPENDENCIES_CYPHER_DIR}/Label_external_types_and_annotations.cypher"
 
-# Preparation - Add Java Artifact node properties "incomingDependencies" and "outgoingDependencies"
+# Preparation - Add Java Artifact node properties "incomingDependencies", "outgoingDependencies" and "version"
 execute_cypher_summarized "${ARTIFACT_DEPENDENCIES_CYPHER_DIR}/Incoming_Java_Artifact_Dependencies.cypher"
 execute_cypher_summarized "${ARTIFACT_DEPENDENCIES_CYPHER_DIR}/Outgoing_Java_Artifact_Dependencies.cypher"
+execute_cypher_summarized "${ARTIFACT_DEPENDENCIES_CYPHER_DIR}/Set_maven_artifact_version.cypher"
 
 # Preparation - Add Java Type node properties "incomingDependencies" and "outgoingDependencies"
 execute_cypher_summarized "${METRICS_CYPHER_DIR}/Set_Incoming_Java_Type_Dependencies.cypher"