diff --git a/.github/workflows/code-reports.yml b/.github/workflows/code-reports.yml index d6743896e..a67a2ab94 100644 --- a/.github/workflows/code-reports.yml +++ b/.github/workflows/code-reports.yml @@ -42,7 +42,7 @@ jobs: matrix: include: - os: ubuntu-latest - java: 11 + java: 17 python: 3.11 mambaforge: 23.1.0-1 diff --git a/COMMANDS.md b/COMMANDS.md index ebd06b020..457e6bdb3 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -4,20 +4,27 @@ Use the following commands in the root directory of this repository to start an analysis manually e.g. for [AxonFramework](./scripts/artifacts/downloadAxonFramework.sh). -### Notes - -- Be sure to use Java 11 (June 2023 Neo4j 4.x requirement) -- Use your own initial Neo4j password -- It uses the script [analyze.sh](./scripts/analysis/analyze.sh) -- The script file names (without the prefix "download" and without the file extension) in the directory [scripts/artifacts](./scripts/artifacts) provide the possible analysis names. - - ```shell export NEO4J_INITIAL_PASSWORD=theinitialpasswordthatihavechosenforneo4j mkdir -p ./temp -./../scripts/analysis/analyze.sh --name AxonFramework --version 4.7.5 --report All +cd temp +./../scripts/analysis/analyze.sh --name AxonFramework --version 4.8.0 ``` +Add the command line argument `--report Csv` to only run the CSV reports when you don't have Python set up +or want to skip Jupyter Notebooks. + +Add the command line argument `--profile Neo4jv4` if you want to use the older long term support (june 2023) +version v4.4.x of Neo4j and compatible versions of plugins and JQAssistant. + +### Notes + +- Be sure to use Java 11 (Mai 2023 Neo4j v4 requirement) or Java 17 (June 2023 Neo4j v5 and jQAssistant CLI v2) +- Use your own initial Neo4j password +- For more details have a look at the script [analyze.sh](./scripts/analysis/analyze.sh) +- The script file names (without the prefix "download" and without the file extension) in the directory [scripts/artifacts](./scripts/artifacts) provide all analysis names that are available. + + Have a look at [code-reports.yml](./.github/workflows/code-reports.yml) for all details about setup steps and full automation. ## Generate Markdown References diff --git a/JQAssistantGraphDatabaseCompatibilityIssues.md b/JQAssistantGraphDatabaseCompatibilityIssues.md new file mode 100644 index 000000000..7489f6981 --- /dev/null +++ b/JQAssistantGraphDatabaseCompatibilityIssues.md @@ -0,0 +1,161 @@ +# jQAssistant Graph Database Compatibility Issues + +## Neo4j v5 + +As it looks like, compatibility to Neo4j v5 is already in progress: [upgraded initial Neo4j queries to support 5.x](https://github.com/buschmais/extended-objects/commit/bd484e283f44a26c250cd80d5108516fa8ca98d0). It will be available with version 2.2 of [com.buschmais.xo - xo.impl](https://mvnrepository.com/artifact/com.buschmais.xo/xo.impl). Then it also needs to be updated in the upstream jQAssistant command line artifact. + +### References + +- https://neo4j.com/docs/upgrade-migration-guide/current/version-5/changelogs/procedures +- https://neo4j.com/docs/operations-manual/current/reference/procedures/#procedure_dbms_components +- https://github.com/buschmais/extended-objects/blob/0b91a5d55c9a76adba28ddccd6caba699eeaa1fc/neo4j/spi/src/main/java/com/buschmais/xo/neo4j/spi/AbstractNeo4jDatastoreSession.java#L66 +- https://github.com/buschmais/extended-objects/commit/bd484e283f44a26c250cd80d5108516fa8ca98d0 + +### Stacktrace + +```java +Exception in thread "main" com.buschmais.xo.api.XOException: Cannot execute statement 'CALL db.indexes() YIELD tokenNames AS labels, properties AS properties RETURN labels, properties', {} + at com.buschmais.xo.neo4j.remote.impl.datastore.StatementExecutor.execute(StatementExecutor.java:51) + at com.buschmais.xo.neo4j.remote.impl.datastore.RemoteCypherQuery.execute(RemoteCypherQuery.java:36) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastoreSession.getIndexes(AbstractNeo4jDatastoreSession.java:70) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastoreSession.getIndexes(AbstractNeo4jDatastoreSession.java:30) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastore.lambda$init$2(AbstractNeo4jDatastore.java:40) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastore.inTransaction(AbstractNeo4jDatastore.java:57) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastore.init(AbstractNeo4jDatastore.java:40) + at com.buschmais.xo.impl.XOManagerFactoryImpl.(XOManagerFactoryImpl.java:57) + at com.buschmais.xo.impl.bootstrap.XOBootstrapServiceImpl.createXOManagerFactory(XOBootstrapServiceImpl.java:41) + at com.buschmais.xo.api.bootstrap.XO.createXOManagerFactory(XO.java:48) + at com.buschmais.jqassistant.core.store.impl.AbstractGraphStore.start(AbstractGraphStore.java:66) + at com.buschmais.jqassistant.commandline.task.AbstractStoreTask.withStore(AbstractStoreTask.java:69) + at com.buschmais.jqassistant.commandline.task.AnalyzeTask.run(AnalyzeTask.java:65) + at com.buschmais.jqassistant.commandline.Main.executeTask(Main.java:306) + at com.buschmais.jqassistant.commandline.Main.executeTasks(Main.java:264) + at com.buschmais.jqassistant.commandline.Main.interpretCommandLine(Main.java:247) + at com.buschmais.jqassistant.commandline.Main.run(Main.java:93) + at com.buschmais.jqassistant.commandline.Main.main(Main.java:64) +Caused by: org.neo4j.driver.exceptions.ClientException: There is no procedure with the name `db.indexes` registered for this database instance. Please ensure you've spelled the procedure name correctly and that the procedure is properly deployed. + at org.neo4j.driver.internal.util.Futures.blockingGet(Futures.java:144) + at org.neo4j.driver.internal.InternalTransaction.run(InternalTransaction.java:60) + at org.neo4j.driver.internal.AbstractQueryRunner.run(AbstractQueryRunner.java:37) + at org.neo4j.driver.internal.AbstractQueryRunner.run(AbstractQueryRunner.java:43) + at com.buschmais.xo.neo4j.remote.impl.datastore.StatementExecutor.execute(StatementExecutor.java:49) + ... 17 more + Suppressed: org.neo4j.driver.internal.util.ErrorUtil$InternalExceptionCause + at org.neo4j.driver.internal.util.ErrorUtil.newNeo4jError(ErrorUtil.java:99) + at org.neo4j.driver.internal.async.inbound.InboundMessageDispatcher.handleFailureMessage(InboundMessageDispatcher.java:122) + at org.neo4j.driver.internal.messaging.common.CommonMessageReader.unpackFailureMessage(CommonMessageReader.java:83) + at org.neo4j.driver.internal.messaging.common.CommonMessageReader.read(CommonMessageReader.java:59) + at org.neo4j.driver.internal.async.inbound.InboundMessageHandler.channelRead0(InboundMessageHandler.java:83) + at org.neo4j.driver.internal.async.inbound.InboundMessageHandler.channelRead0(InboundMessageHandler.java:35) + at org.neo4j.driver.internal.shaded.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) + at org.neo4j.driver.internal.async.inbound.MessageDecoder.channelRead(MessageDecoder.java:47) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:311) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:432) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) + at org.neo4j.driver.internal.shaded.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) + at org.neo4j.driver.internal.shaded.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) + at org.neo4j.driver.internal.shaded.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:655) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:581) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) + at org.neo4j.driver.internal.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) + at org.neo4j.driver.internal.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) + at org.neo4j.driver.internal.shaded.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) + at java.base/java.lang.Thread.run(Thread.java:833) +``` + +## Memgraph + +The method [AbstractNeo4jDatastoreSession.getNeo4jVersion](https://github.com/buschmais/extended-objects/blob/0b91a5d55c9a76adba28ddccd6caba699eeaa1fc/neo4j/spi/src/main/java/com/buschmais/xo/neo4j/spi/AbstractNeo4jDatastoreSession.java#L50) uses the procedure `dbms.components()`. + +This procedure is available in Neo4j v4, doesn't seem to be there in memgraph. + +> List DBMS components and their versions. +> dbms.components() :: (name :: STRING?, versions :: LIST? OF STRING?, edition :: STRING?) + +### Stacktrace + +```java +Exception in thread "main" com.buschmais.xo.api.XOException: Cannot execute statement 'CALL dbms.components() YIELD versions UNWIND versions AS version RETURN version', {} + at com.buschmais.xo.neo4j.remote.impl.datastore.StatementExecutor.execute(StatementExecutor.java:51) + at com.buschmais.xo.neo4j.remote.impl.datastore.RemoteCypherQuery.execute(RemoteCypherQuery.java:36) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastoreSession.getNeo4jVersion(AbstractNeo4jDatastoreSession.java:51) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastoreSession.getIndexes(AbstractNeo4jDatastoreSession.java:29) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastore.lambda$init$2(AbstractNeo4jDatastore.java:40) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastore.inTransaction(AbstractNeo4jDatastore.java:57) + at com.buschmais.xo.neo4j.spi.AbstractNeo4jDatastore.init(AbstractNeo4jDatastore.java:40) + at com.buschmais.xo.impl.XOManagerFactoryImpl.(XOManagerFactoryImpl.java:57) + at com.buschmais.xo.impl.bootstrap.XOBootstrapServiceImpl.createXOManagerFactory(XOBootstrapServiceImpl.java:41) + at com.buschmais.xo.api.bootstrap.XO.createXOManagerFactory(XO.java:48) + at com.buschmais.jqassistant.core.store.impl.AbstractGraphStore.start(AbstractGraphStore.java:66) + at com.buschmais.jqassistant.commandline.task.AbstractStoreTask.withStore(AbstractStoreTask.java:69) + at com.buschmais.jqassistant.commandline.task.ScanTask.run(ScanTask.java:78) + at com.buschmais.jqassistant.commandline.Main.executeTask(Main.java:306) + at com.buschmais.jqassistant.commandline.Main.executeTasks(Main.java:264) + at com.buschmais.jqassistant.commandline.Main.interpretCommandLine(Main.java:247) + at com.buschmais.jqassistant.commandline.Main.run(Main.java:93) + at com.buschmais.jqassistant.commandline.Main.main(Main.java:64) +Caused by: org.neo4j.driver.exceptions.ClientException: There is no procedure named 'dbms.components'. + at org.neo4j.driver.internal.util.Futures.blockingGet(Futures.java:144) + at org.neo4j.driver.internal.InternalTransaction.run(InternalTransaction.java:60) + at org.neo4j.driver.internal.AbstractQueryRunner.run(AbstractQueryRunner.java:37) + at org.neo4j.driver.internal.AbstractQueryRunner.run(AbstractQueryRunner.java:43) + at com.buschmais.xo.neo4j.remote.impl.datastore.StatementExecutor.execute(StatementExecutor.java:49) + ... 17 more + Suppressed: org.neo4j.driver.internal.util.ErrorUtil$InternalExceptionCause + at org.neo4j.driver.internal.util.ErrorUtil.newNeo4jError(ErrorUtil.java:99) + at org.neo4j.driver.internal.async.inbound.InboundMessageDispatcher.handleFailureMessage(InboundMessageDispatcher.java:122) + at org.neo4j.driver.internal.messaging.common.CommonMessageReader.unpackFailureMessage(CommonMessageReader.java:83) + at org.neo4j.driver.internal.messaging.common.CommonMessageReader.read(CommonMessageReader.java:59) + at org.neo4j.driver.internal.async.inbound.InboundMessageHandler.channelRead0(InboundMessageHandler.java:83) + at org.neo4j.driver.internal.async.inbound.InboundMessageHandler.channelRead0(InboundMessageHandler.java:35) + at org.neo4j.driver.internal.shaded.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) + at org.neo4j.driver.internal.async.inbound.MessageDecoder.channelRead(MessageDecoder.java:47) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:311) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:432) + at org.neo4j.driver.internal.shaded.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) + at org.neo4j.driver.internal.shaded.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) + at org.neo4j.driver.internal.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) + at org.neo4j.driver.internal.shaded.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:655) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:581) + at org.neo4j.driver.internal.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) + at org.neo4j.driver.internal.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) + at org.neo4j.driver.internal.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) + at org.neo4j.driver.internal.shaded.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) + at java.base/java.lang.Thread.run(Thread.java:829) +``` \ No newline at end of file diff --git a/README.md b/README.md index 3529b3cfe..d62391262 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,9 @@ The [Code Reports Pipeline](./.github/workflows/code-reports.yml) utilizes [GitH ## 🤔 Questions & Answers +- How can i run an analysis locally? +👉 See [start-an-analysis](./COMMANDS.md#start-an-analysis) in the [Commands Reference](./COMMANDS.md). + - How can i add an CSV report to the pipeline? 👉 Put your new cypher query into the [cypher](./cypher) directory or a suitable (new) sub directory. 👉 Create a new CSV report script in the [scripts/reports](./scripts/reports/) directory. Take for example [OverviewCsv.sh](./scripts/reports/OverviewCsv.sh) as a reference. diff --git a/cypher/Count_nodes_and_relationships.cypher b/cypher/Count_nodes_and_relationships.cypher new file mode 100644 index 000000000..c571e03e8 --- /dev/null +++ b/cypher/Count_nodes_and_relationships.cypher @@ -0,0 +1,5 @@ +// Count nodes and relationships + +MATCH (n)-[r]-(m) +RETURN COUNT(DISTINCT n) AS nodeCount + ,COUNT(DISTINCT r) AS relationshipCount \ No newline at end of file diff --git a/cypher/Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_ARTIFACT.cypher b/cypher/Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_ARTIFACT.cypher new file mode 100644 index 000000000..56fde59b4 --- /dev/null +++ b/cypher/Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_ARTIFACT.cypher @@ -0,0 +1,7 @@ +// Create a DEPENDS_ON relationship for every DEPENDS_ON_ARTIFACT + +MATCH (a:Artifact)-[existing:DEPENDS_ON_ARTIFACT]->(b:Artifact) +MERGE (a)-[created:DEPENDS_ON]->(b) + WITH count(existing) as numberOfExistingRelations + ,count(created) as numberOfCreatedRelations +RETURN numberOfExistingRelations, numberOfCreatedRelations \ No newline at end of file diff --git a/cypher/Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_PACKAGE.cypher b/cypher/Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_PACKAGE.cypher new file mode 100644 index 000000000..d051f10dd --- /dev/null +++ b/cypher/Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_PACKAGE.cypher @@ -0,0 +1,7 @@ +// Create a DEPENDS_ON relationship for every DEPENDS_ON_PACKAGE + +MATCH (a:Package)-[existing:DEPENDS_ON_PACKAGE]->(b:Package) +MERGE (a)-[created:DEPENDS_ON]->(b) + WITH count(existing) as numberOfExistingRelations + ,count(created) as numberOfCreatedRelations +RETURN numberOfExistingRelations, numberOfCreatedRelations \ No newline at end of file diff --git a/scripts/SCRIPTS.md b/scripts/SCRIPTS.md index 2daa5c1d3..c1cf0011b 100644 --- a/scripts/SCRIPTS.md +++ b/scripts/SCRIPTS.md @@ -16,8 +16,13 @@ Script | Directory | Description | [generateCypherReference.sh](./generateCypherReference.sh) | | Generates "CYPHER.md" containing a reference to all Cypher files in this directory and its subdirectories. | | [generateMarkdownReference.sh](./generateMarkdownReference.sh) | | Generates "REPORTS.md" containing a reference to all scripts in this directory and its subdirectories. | | [generateScriptReference.sh](./generateScriptReference.sh) | | Generates "SCRIPTS.md" containing a reference to all scripts in this directory and its subdirectories. | +| [prepareAnalysis.sh](./prepareAnalysis.sh) | | Prepares and validates the graph database before analysis | +| [Default.sh](./profiles/Default.sh) | profiles | Sets (if any) settings variables for a default analysis. | +| [Neo4jv4.sh](./profiles/Neo4jv4.sh) | profiles | Sets all settings variables for an analysis with Neo4j v4.4.x (long term support (LTS) version as of may 2023). | +| [Neo4jv5.sh](./profiles/Neo4jv5.sh) | profiles | Sets all settings variables for an analysis with Neo4j v5.x (newest version as of june 2023). | | [CentralityCsv.sh](./reports/CentralityCsv.sh) | reports | Looks for centrality using the Graph Data Science Library of Neo4j and creates CSV reports. | | [CommunityCsv.sh](./reports/CommunityCsv.sh) | reports | Detects communities using the Graph Data Science Library of Neo4j and creates CSV reports. | +| [DatabaseCsvExport.sh](./reports/DatabaseCsvExport.sh) | reports | Exports the whole graph database as a CSV file using the APOC procedure "apoc.export.csv.all" | | [ExternalDependenciesCsv.sh](./reports/ExternalDependenciesCsv.sh) | reports | Executes "Package_Usage" Cypher queries to get the "package-dependencies" CSV reports. | | [ExternalDependenciesJupyter.sh](./reports/ExternalDependenciesJupyter.sh) | reports | Creates the "overview" report (ipynb, md, pdf) based on the Jupyter Notebook "Overview.ipynb". | | [ObjectOrientedDesignMetricsCsv.sh](./reports/ObjectOrientedDesignMetricsCsv.sh) | reports | Executes "Metrics" Cypher queries to get the "object-oriented-design-metrics" CSV reports. | @@ -35,6 +40,7 @@ Script | Directory | Description | [JupyterReports.sh](./reports/compilations/JupyterReports.sh) | compilations | Runs all Jupyter Notebook report scripts. | | [resetAndScan.sh](./resetAndScan.sh) | | Deletes all data in the Neo4j graph database and rescans the downloaded artifacts to create a new graph. | | [resetAndScanChanged.sh](./resetAndScanChanged.sh) | | Executes "resetAndScan.sh" only if "detectChangedArtifacts.sh" returns detected changes. | +| [resetAndScanMemgraph.sh](./resetAndScanMemgraph.sh) | | Deletes all data in the Neo4j graph database and rescans the downloaded artifacts to create a new graph. | | [setupJQAssistant.sh](./setupJQAssistant.sh) | | Installs (download and unzip) jQAssistant (https://jqassistant.org/get-started). | | [setupNeo4j.sh](./setupNeo4j.sh) | | Installs (download, unpack, get plugins, configure) a local Neo4j Graph Database (https://neo4j.com/download-center/#community). | | [setupNeo4jInitialPassword.sh](./setupNeo4jInitialPassword.sh) | | Sets the initial password for the local Neo4j Graph Database (https://neo4j.com/download-center/#community). | diff --git a/scripts/analysis/analyze.sh b/scripts/analysis/analyze.sh index 81a163160..c7366c929 100755 --- a/scripts/analysis/analyze.sh +++ b/scripts/analysis/analyze.sh @@ -10,39 +10,42 @@ # It is recommended to create an empty directory (preferrable "temp") # and change into it prior to starting this script. -# Note: The first argument "--name" is requried. It is used to create the working directory +# Note: The argument "--name" is requried. It is used to create the working directory # as well as to find the script "scripts/artifacts/download.sh" to download the artifacts. -# Note: The second argument "--version" is optional. +# Note: The argument "--version" is also required. # It is appended to the working directory to further distinguish the results. +# The version is passed to the artifacts download script as an argument. + +# Note: The argument "--report" is optional. The default value is "All". +# It selects the report compilation, a named group of reports. Besides the default "All" there are e.g. "Csv" and "Jupyter". +# This makes it possible to run only a subset of the reports. For example "Csv" won't need python to be set up and runs therefore much faster. +# Implemented is this as a script in "scripts/reports/compilations" that starts with the report compilation name followed by "Reports.sh". + +# Note: The argument "--profile" is optional. The default value is "Default". +# It selects a settings profile that sets all suitable variables for the analysis. +# This makes it possible to run an analysis with e.g. Neo4j v4 instead of v5. Further profiles might come in future. +# Implemented is this as a script in "scripts/profiles" that starts with the settings profile name followed by ".sh". # Note: The script and its sub scripts are designed to be as efficient as possible # when it comes to subsequent executions. # Existing downloads, installations, artifacts, scans and processes will be detected. -# Overrideable environment variables (optional, defaults also defined in sub scripts where needed) -NEO4J_EDITION=${NEO4J_EDITION:-"community"} # Choose "community" or "enterprise" -NEO4J_VERSION=${NEO4J_VERSION:-"4.4.20"} # Version 4.4.x is the current long term support (LTS) version (may 2023) -JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"1.12.2"} # Version 1.12.2 is the current version (may 2023) - -# Overrideable environment variables for ports (optional, defaults also defined in sub scripts where needed) -# Override them if you need to run multiple neo4j database servers in parallel. -NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} -NEO4J_HTTPS_PORT=${NEO4J_HTTPS_PORT:-"7473"} -NEO4J_BOLT_PORT=${NEO4J_BOLT_PORT:-"7687"} - +# Overrideable variables with directory names ARTIFACT_SCRIPTS_DIRECTORY=${ARTIFACT_SCRIPTS_DIRECTORY:-"artifacts"} REPORTS_SCRIPTS_DIRECTORY=${REPORTS_SCRIPTS_DIRECTORY:-"reports"} REPORT_COMPILATIONS_SCRIPTS_DIRECTORY=${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY:-"compilations"} +SETTINGS_PROFILE_SCRIPTS_DIRECTORY=${SETTINGS_PROFILE_SCRIPTS_DIRECTORY:-"profiles"} # Function to display script usage usage() { - echo "Usage: $0 --name --version [--report ]" + echo "Usage: $0 --name --version [--report ] [--profile ]" exit 1 } # Default values analysisReportCompilation="All" +settingsProfile="Default" # Parse command line arguments while [[ $# -gt 0 ]]; do @@ -60,8 +63,12 @@ while [[ $# -gt 0 ]]; do analysisReportCompilation="$2" shift ;; + --profile) + settingsProfile="$2" + shift + ;; *) - echo "Error (analyze): Unknown option: ${key}" + echo "analyze: Error: Unknown option: ${key}" usage ;; esac @@ -70,25 +77,31 @@ done # Check if the name is provided if [[ -z ${analysisName} ]]; then - echo "Error (analyze): Name is required." + echo "analyze ${analysisName}: Error: Name is required." usage fi # Check if the version is provided if [[ -z ${analysisVersion} ]]; then - echo "Error (analyze): Version is required." + echo "analyze ${analysisName}: Error: Version is required." usage fi # Assure that the analysis name only consists of letters and numbers if ! [[ ${analysisName} =~ ^[[:alnum:]]+$ ]]; then - echo "Error (analyze): Name can only contain letters and numbers." + echo "analyze ${analysisName}: Error: Name can only contain letters and numbers." exit 1 fi # Assure that the analysis report compilation only consists of letters and numbers if ! [[ ${analysisReportCompilation} =~ ^[[:alnum:]]+$ ]]; then - echo "Error (analyze): Report can only contain letters and numbers." + echo "analyze ${analysisName}: Report can only contain letters and numbers." + exit 1 +fi + +# Assure that the settings profile only consists of letters and numbers +if ! [[ ${settingsProfile} =~ ^[[:alnum:]]+$ ]]; then + echo "analyze ${analysisName}: Error: Settings profile can only contain letters and numbers." exit 1 fi @@ -103,26 +116,31 @@ echo "analyze ${analysisName}: ANALYSIS_SCRIPT_DIR=${ANALYSIS_SCRIPT_DIR}" SCRIPTS_DIR=${SCRIPTS_DIR:-$(dirname -- "${ANALYSIS_SCRIPT_DIR}")} echo "analyze ${analysisName}: SCRIPTS_DIR=${SCRIPTS_DIR}" -# Assure that there is a download script for the given analysis name. -if [ ! -f "${SCRIPTS_DIR}/${ARTIFACT_SCRIPTS_DIRECTORY}/download${analysisName}.sh" ] ; then - echo "Error (analyze): No download${analysisName}.sh script in the directory ${SCRIPTS_DIR}/${ARTIFACT_SCRIPTS_DIRECTORY} for analysis name ${analysisName}." - exit 1 -fi - # Assure that there is a download script for the given analysis name argument. DOWNLOAD_SCRIPT="${SCRIPTS_DIR}/${ARTIFACT_SCRIPTS_DIRECTORY}/download${analysisName}.sh" if [ ! -f "${DOWNLOAD_SCRIPT}" ] ; then - echo "Error (analyze): No download${analysisName}.sh script in the directory ${SCRIPTS_DIR}/${ARTIFACT_SCRIPTS_DIRECTORY} for analysis name ${analysisName}." + echo "analyze ${analysisName}: Error: No download${analysisName}.sh script in the directory ${SCRIPTS_DIR}/${ARTIFACT_SCRIPTS_DIRECTORY} for analysis name ${analysisName}." exit 1 fi # Assure that there is a report compilation script for the given report argument. REPORT_COMPILATION_SCRIPT="${SCRIPTS_DIR}/${REPORTS_SCRIPTS_DIRECTORY}/${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY}/${analysisReportCompilation}Reports.sh" if [ ! -f "${REPORT_COMPILATION_SCRIPT}" ] ; then - echo "Error (analyze): No ${analysisReportCompilation}Reports.sh script in the directory ${SCRIPTS_DIR}/${REPORTS_SCRIPTS_DIRECTORY}/${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY} for report name ${analysisReportCompilation}." + echo "analyze ${analysisName}: Error: No ${analysisReportCompilation}Reports.sh script in the directory ${SCRIPTS_DIR}/${REPORTS_SCRIPTS_DIRECTORY}/${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY} for report name ${analysisReportCompilation}." + exit 1 +fi + +# Assure that there is a script file for the given settings profile name. +SETTINGS_PROFILE_SCRIPT="${SCRIPTS_DIR}/${SETTINGS_PROFILE_SCRIPTS_DIRECTORY}/${settingsProfile}.sh" +if [ ! -f "${SETTINGS_PROFILE_SCRIPT}" ] ; then + echo "analyze ${analysisName}: Error: No ${settingsProfile}.sh script in the directory ${SCRIPTS_DIR}/${SETTINGS_PROFILE_SCRIPTS_DIRECTORY} for settings profile ${settingsProfile}." exit 1 fi +# Execute the settings profile script that sets all the neccessary settings variables (overrideable by environment variables). +echo "analyze ${analysisName}: Using analysis settings profile script ${SETTINGS_PROFILE_SCRIPT}" +source "${SETTINGS_PROFILE_SCRIPT}" || exit 1 + # Create working directory if it hadn't been created yet workingDirectory="${analysisName}-${analysisVersion}" mkdir -p "${workingDirectory}" || exit 2 # Create the working directory only if it doesn't exist @@ -141,14 +159,17 @@ source "${DOWNLOAD_SCRIPT}" "${analysisVersion}" || exit 4 # Scan and analyze artifacts when they were changed source "${SCRIPTS_DIR}/resetAndScanChanged.sh" || exit 5 +# Prepare and validate graph database before analyzing and creating reports +source "${SCRIPTS_DIR}/prepareAnalysis.sh" || exit 6 + ######################### # Create Reports ######################### echo "Creating Reports with ${REPORT_COMPILATION_SCRIPT} ..." -source "${REPORT_COMPILATION_SCRIPT}" || exit 6 +source "${REPORT_COMPILATION_SCRIPT}" || exit 7 # Stop Neo4j at the end source "${SCRIPTS_DIR}/stopNeo4j.sh" # Change back to the previous directory where the script was started -popd || exit 7 \ No newline at end of file +popd || exit 8 \ No newline at end of file diff --git a/scripts/executeQuery.sh b/scripts/executeQuery.sh index 04f690207..f27980875 100755 --- a/scripts/executeQuery.sh +++ b/scripts/executeQuery.sh @@ -15,6 +15,7 @@ # Overrideable Defaults NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} +NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/neo4j/tx/commit"} # Neo4j v5: "db//tx/commit", Neo4j v4: "db/data/transaction/commit", # Check if environment variable is set if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then @@ -85,7 +86,7 @@ cypher_query_for_api="{\"statements\":[{\"statement\":${cypher_query},\"includeS # Calls the Neo4j HTTP API using cURL ( https://curl.se ) cyper_query_result=$(curl --silent -S --fail-with-body -H Accept:application/json -H Content-Type:application/json \ -u neo4j:"${NEO4J_INITIAL_PASSWORD}" \ - "http://localhost:${NEO4J_HTTP_PORT}/db/data/transaction/commit" \ + "http://localhost:${NEO4J_HTTP_PORT}/${NEO4J_HTTP_TRANSACTION_ENDPOINT}" \ -d "${cypher_query_for_api}") #echo "Cypher Query Result: $cyper_query_result" diff --git a/scripts/executeQueryFunctions.sh b/scripts/executeQueryFunctions.sh index c596dddb7..472385634 100644 --- a/scripts/executeQueryFunctions.sh +++ b/scripts/executeQueryFunctions.sh @@ -8,14 +8,19 @@ # This way non-standard tools like readlink aren't needed. SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} -# Function to execute a cypher query from the given file (first and only argument) with the default method (HTTP) +# Function to execute a cypher query from the given file (first argument) with the default method execute_cypher() { - execute_cypher_http "${1}" + execute_cypher_http "${1}" || exit 1 } -# Function to execute a cypher query from the given file (first and only argument) with the default method (HTTP) +# Function to execute a cypher query from the given file (first argument) with the default method and just return the number of results execute_cypher_summarized() { - execute_cypher_http_summarized "${1}" + execute_cypher_http_summarized "${1}" || exit 1 +} + +# Function to execute a cypher query from the given file (first argument) with the default method and fail if there is no result +execute_cypher_expect_results() { + execute_cypher_http_expect_results "${1}" || exit 1 } # Function to execute a cypher query from the given file (first and only argument) using Neo4j's HTTP API @@ -34,23 +39,30 @@ execute_cypher_http_summarized() { results=$( execute_cypher_http ${cypherFileName} | wc -l ) results=$((results - 2)) - echo "$(basename -- "${cypherFileName}") (http) result lines: ${results}" + echo "$(basename -- "${cypherFileName}") (via http) result lines: ${results}" } -# Function to execute a cypher query from the given file (first and only argument) using "cypher-shell" provided by Neo4j -execute_cypher_shell() { +# Function to execute a cypher query from the given file (first and only argument) that fails on no result using Neo4j's HTTP API +execute_cypher_http_expect_results() { # Get the Cypher file name from the first argument - cypherFileName=$1 + cypherFileName="${1}" - # Check if NEO4J_DIRECTORY exists - if [ ! -d "${NEO4J_DIRECTORY}" ] ; then - echo "Neo4j Installation Directory <${NEO4J_DIRECTORY}> doesn't exist. Please run setupNeo4j.sh first." + results=$( execute_cypher_http ${cypherFileName} | wc -l ) + results=$((results - 1)) + if [[ "$results" -lt 1 ]]; then + echo "$(basename -- "${cypherFileName}") (via http) Error: Expected at least one entry but was ${results}" exit 1 fi +} + +# Function to execute a cypher query from the given file (first and only argument) using "cypher-shell" provided by Neo4j +execute_cypher_shell() { + # Get the Cypher file name from the first argument + cypherFileName=$1 # Check if NEO4J_BIN exists if [ ! -d "${NEO4J_BIN}" ] ; then - echo "Neo4j Binary Directory <${NEO4J_BIN}> doesn't exist. Please run setupNeo4j.sh first." + echo "executeQuery: Error: Neo4j Binary Directory <${NEO4J_BIN}> doesn't exist. Please run setupNeo4j.sh first." exit 1 fi @@ -64,11 +76,24 @@ execute_cypher_shell() { } # Function to execute a cypher query from the given file (first and only argument) with a summarized (console) output using "cypher-shell" provided by Neo4j -execute_cypher_summarized_shell() { +execute_cypher_shell_summarized() { # Get the Cypher file name from the first argument cypherFileName=$1 results=$( execute_cypher_shell ${cypherFileName} | wc -l ) results=$((results - 2)) - echo "$(basename -- "${cypherFileName}") (cypher-shell) result lines: ${results}" || exit 1 + echo "$(basename -- "${cypherFileName}") (via cypher-shell) result lines: ${results}" || exit 1 +} + +# Function to execute a cypher query from the given file (first and only argument) that fails on no result using "cypher-shell" provided by Neo4j +execute_cypher_shell_expect_results() { + # Get the Cypher file name from the first argument + cypherFileName=$1 + + results=$( execute_cypher_shell ${cypherFileName} | wc -l ) + results=$((results - 2)) + if [[ "$results" -lt 1 ]]; then + echo "$(basename -- "${cypherFileName}") (via cypher-shell) Error: Expected at least one entry but was ${results}" + exit 1 + fi } \ No newline at end of file diff --git a/scripts/prepareAnalysis.sh b/scripts/prepareAnalysis.sh new file mode 100644 index 000000000..e3f31e3f5 --- /dev/null +++ b/scripts/prepareAnalysis.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Prepares and validates the graph database before analysis + +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} +echo "prepareAnalysis: SCRIPTS_DIR=${SCRIPTS_DIR}" + +# Check if environment variable is set +if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then + echo "prepareAnalysis: Error: Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD='." + exit 1 +fi + +# Get the "cypher" directory by taking the path of this script, going one directory up and then into "cypher". +CYPHER_DIR=${CYPHER_DIR:-"${SCRIPTS_DIR}/../cypher"} +echo "prepareAnalysis: CYPHER_DIR=$CYPHER_DIR" + +# Define functions to execute a cypher query from within the given file (first and only argument) +source "${SCRIPTS_DIR}/executeQueryFunctions.sh" + +# Local Constants +PACKAGE_WEIGHTS_CYPHER_DIR="$CYPHER_DIR/Package_Relationship_Weights" +PACKAGE_METRICS_CYPHER_DIR="$CYPHER_DIR/Metrics" + +# Preparation - Create DEPENDS_ON for every DEPENDS_ON_PACKAGE relationship +execute_cypher_expect_results "${CYPHER_DIR}/Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_PACKAGE.cypher" || exit 1 +execute_cypher_expect_results "${CYPHER_DIR}/Create_a_DEPENDS_ON_relationship_for_every_DEPENDS_ON_ARTIFACT.cypher" || exit 1 + +# Preparation - Add weights to package DEPENDS_ON relationships +execute_cypher_expect_results "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight_property_for_Interface_Dependencies_to_Package_DEPENDS_ON_Relationship.cypher" || exit 1 +execute_cypher_expect_results "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight_property_to_Package_DEPENDS_ON_Relationship.cypher" || exit 1 +execute_cypher_expect_results "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight25PercentInterfaces_to_Package_DEPENDS_ON_relationships.cypher" || exit 1 +execute_cypher_expect_results "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight10PercentInterfaces_to_Package_DEPENDS_ON_relationships.cypher" || exit 1 + +# Preparation - Add Package node properties "incomingDependencies" and "outgoingDependencies" +execute_cypher_expect_results "${PACKAGE_METRICS_CYPHER_DIR}/Set_Incoming_Package_Dependencies.cypher" || exit 1 +execute_cypher_expect_results "${PACKAGE_METRICS_CYPHER_DIR}/Set_Outgoing_Package_Dependencies.cypher" || exit 1 diff --git a/scripts/profiles/Default.sh b/scripts/profiles/Default.sh new file mode 100755 index 000000000..62b8fda6d --- /dev/null +++ b/scripts/profiles/Default.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Sets (if any) settings variables for a default analysis. +# The chosen settings are tested to be compatible and working. \ No newline at end of file diff --git a/scripts/profiles/Neo4jv4.sh b/scripts/profiles/Neo4jv4.sh new file mode 100755 index 000000000..a7caaaa0c --- /dev/null +++ b/scripts/profiles/Neo4jv4.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Sets all settings variables for an analysis with Neo4j v4.4.x (long term support (LTS) version as of may 2023). +# The chosen settings are tested to be compatible and working. + +NEO4J_VERSION=${NEO4J_VERSION:-"4.4.20"} # Version 4.4.x is the current long term support (LTS) version (may 2023) +NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/data/transaction/commit"} # Since Neo4j v5 it is "db//tx/commit" + +# Overrideable settings variables for ports (optional, defaults also defined in sub scripts where needed) +# Override them if you need to run multiple neo4j database servers in parallel. +NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} +NEO4J_HTTPS_PORT=${NEO4J_HTTPS_PORT:-"7473"} +NEO4J_BOLT_PORT=${NEO4J_BOLT_PORT:-"7687"} + +# Awesome Procedures (APOC) Plugin for Neo4j +NEO4J_APOC_PLUGIN_VERSION=${NEO4J_APOC_PLUGIN_VERSION:-"4.4.0.15"} # Version number matches Neo4j version +NEO4J_APOC_PLUGIN_EDITION=${NEO4J_APOC_PLUGIN_EDITION:-"all"} # Since Neo4j v5 only the core edition is maintained +NEO4J_APOC_PLUGIN_GITHUB=${NEO4J_APOC_PLUGIN_GITHUB:-"neo4j-contrib/neo4j-apoc-procedures"} # Location for the old plugins compatible to Neo4j v4 + +NEO4J_GDS_PLUGIN_VERSION=${NEO4J_GDS_PLUGIN_VERSION:-"2.3.4"} # Graph Data Science Plugin Version 2.3.x is compatible with Neo4j 4.4.x + +JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"1.12.2"} # Version 1.12.2 is the newest version (may 2023) compatible with Neo4j v4 +JQASSISTANT_CLI_ARTIFACT=${JQASSISTANT_CLI_ARTIFACT:-"jqassistant-commandline-neo4jv3"} # For Neo4j v3 & 4: "jqassistant-commandline-neo4jv3" +JQASSISTANT_CLI_DISTRIBUTION=${JQASSISTANT_CLI_DISTRIBUTION:-"distribution.zip"} # Neo4j v3 & 4: "distribution.zip" +JQASSISTANT_CONFIG_TEMPLATE=${JQASSISTANT_CONFIG_TEMPLATE:-"template-neo4jv4-jqassistant.yaml"} \ No newline at end of file diff --git a/scripts/profiles/Neo4jv5.sh b/scripts/profiles/Neo4jv5.sh new file mode 100755 index 000000000..50a4fe215 --- /dev/null +++ b/scripts/profiles/Neo4jv5.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Sets all settings variables for an analysis with Neo4j v5.x (newest version as of june 2023). +# The chosen settings are tested to be compatible and working. + +NEO4J_VERSION=${NEO4J_VERSION:-"5.9.0"} # Version 5.9.0 is the current version of june 2023 +NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/neo4j/tx/commit"} # Since Neo4j v5 it is "db//tx/commit" + +# Overrideable settings variables for ports (optional, defaults also defined in sub scripts where needed) +# Override them if you need to run multiple neo4j database servers in parallel. +NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} +NEO4J_HTTPS_PORT=${NEO4J_HTTPS_PORT:-"7473"} +NEO4J_BOLT_PORT=${NEO4J_BOLT_PORT:-"7687"} + +# Awesome Procedures (APOC) Plugin for Neo4j +NEO4J_APOC_PLUGIN_VERSION=${NEO4J_APOC_PLUGIN_VERSION:-"5.9.0"} # Version number matches Neo4j version since 5.x +NEO4J_APOC_PLUGIN_EDITION=${NEO4J_APOC_PLUGIN_EDITION:-"core"} # Since Neo4j v5 the core edition is updated with Neo4j +NEO4J_APOC_PLUGIN_GITHUB=${NEO4J_APOC_PLUGIN_GITHUB:-"neo4j/apoc"} # Core edition was moved to "neo4j/apoc" for Neo4j v5 + +NEO4J_GDS_PLUGIN_VERSION=${NEO4J_GDS_PLUGIN_VERSION:-"2.4.0"} # Version 2.4.0 is the newest version of june 2023 and compatible with Neo4j v5 + +JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.0.3"} # Version 2.0.3 is the newest version (june 2023) compatible with Neo4j v5 +JQASSISTANT_CLI_ARTIFACT=${JQASSISTANT_CLI_ARTIFACT:-"jqassistant-commandline-distribution"} # Since jQAssistant CLI v2: "jqassistant-commandline-distribution" +JQASSISTANT_CLI_DISTRIBUTION=${JQASSISTANT_CLI_DISTRIBUTION:-"bin.zip"} # Since jQAssistant CLI v2: "bin.zip" +JQASSISTANT_CONFIG_TEMPLATE=${JQASSISTANT_CONFIG_TEMPLATE:-"template-neo4jv5-jqassistant.yaml"} \ No newline at end of file diff --git a/scripts/reports/CentralityCsv.sh b/scripts/reports/CentralityCsv.sh index 7c695b525..5a2a1d1d8 100755 --- a/scripts/reports/CentralityCsv.sh +++ b/scripts/reports/CentralityCsv.sh @@ -3,6 +3,7 @@ # Looks for centrality using the Graph Data Science Library of Neo4j and creates CSV reports. # It requires an already running Neo4j graph database with already scanned analyzed artifacts. # The reports (csv files) will be written into the sub directory reports/community. +# Note that "scripts/prepareAnalysis.sh" is required to run prior to this script. # Overrideable Constants (defaults also defined in sub scripts) REPORTS_DIRECTORY=${REPORTS_DIRECTORY:-"reports"} @@ -31,20 +32,8 @@ FULL_REPORT_DIRECTORY="${REPORTS_DIRECTORY}/${REPORT_NAME}" mkdir -p "${FULL_REPORT_DIRECTORY}" # Local Constants -PACKAGE_WEIGHTS_CYPHER_DIR="$CYPHER_DIR/Package_Relationship_Weights" -PACKAGE_METRICS_CYPHER_DIR="$CYPHER_DIR/Metrics" CENTRALITY_CYPHER_DIR="$CYPHER_DIR/Centrality" -# Preparation - Add weights to package DEPENDS_ON relationships -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight_property_for_Interface_Dependencies_to_Package_DEPENDS_ON_Relationship.cypher" -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight_property_to_Package_DEPENDS_ON_Relationship.cypher" -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight25PercentInterfaces_to_Package_DEPENDS_ON_relationships.cypher" -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight10PercentInterfaces_to_Package_DEPENDS_ON_relationships.cypher" - -# Preparation - Add Package node properties "incomingDependencies" and "outgoingDependencies" -execute_cypher_summarized "${PACKAGE_METRICS_CYPHER_DIR}/Set_Incoming_Package_Dependencies.cypher" -execute_cypher_summarized "${PACKAGE_METRICS_CYPHER_DIR}/Set_Outgoing_Package_Dependencies.cypher" - # Preparation for Centrality - Create package dependencies projection execute_cypher "${CENTRALITY_CYPHER_DIR}/Centrality_0_Delete_Projection.cypher" execute_cypher "${CENTRALITY_CYPHER_DIR}/Centrality_0b_Delete_Subraph_Projection.cypher" diff --git a/scripts/reports/CommunityCsv.sh b/scripts/reports/CommunityCsv.sh index 23cdcd0d3..bb44a3da8 100755 --- a/scripts/reports/CommunityCsv.sh +++ b/scripts/reports/CommunityCsv.sh @@ -3,6 +3,7 @@ # Detects communities using the Graph Data Science Library of Neo4j and creates CSV reports. # It requires an already running Neo4j graph database with already scanned analyzed artifacts. # The reports (csv files) will be written into the sub directory reports/community. +# Note that "scripts/prepareAnalysis.sh" is required to run prior to this script. # Overrideable Constants (defaults also defined in sub scripts) REPORTS_DIRECTORY=${REPORTS_DIRECTORY:-"reports"} @@ -31,21 +32,9 @@ FULL_REPORT_DIRECTORY="${REPORTS_DIRECTORY}/${REPORT_NAME}" mkdir -p "${FULL_REPORT_DIRECTORY}" # Local Constants -PACKAGE_WEIGHTS_CYPHER_DIR="$CYPHER_DIR/Package_Relationship_Weights" -PACKAGE_METRICS_CYPHER_DIR="$CYPHER_DIR/Metrics" LOUVAIN_CYPHER_DIR="$CYPHER_DIR/Community_Detection_Louvain" LEIDEN_CYPHER_DIR="$CYPHER_DIR/Community_Detection_Leiden" -# Preparation - Add weights to package DEPENDS_ON relationships -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight_property_for_Interface_Dependencies_to_Package_DEPENDS_ON_Relationship.cypher" -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight_property_to_Package_DEPENDS_ON_Relationship.cypher" -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight25PercentInterfaces_to_Package_DEPENDS_ON_relationships.cypher" -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight10PercentInterfaces_to_Package_DEPENDS_ON_relationships.cypher" - -# Preparation - Add Package node properties "incomingDependencies" and "outgoingDependencies" -execute_cypher_summarized "${PACKAGE_METRICS_CYPHER_DIR}/Set_Incoming_Package_Dependencies.cypher" -execute_cypher_summarized "${PACKAGE_METRICS_CYPHER_DIR}/Set_Outgoing_Package_Dependencies.cypher" - # Preparation for Community Detection - Create package dependencies projections execute_cypher "${LOUVAIN_CYPHER_DIR}/Community_Detection_0_Delete_Projection.cypher" execute_cypher "${LOUVAIN_CYPHER_DIR}/Community_Detection_0b_Delete_Projection.cypher" diff --git a/scripts/reports/SimilarityCsv.sh b/scripts/reports/SimilarityCsv.sh index cd7201c12..12f7a8c8f 100755 --- a/scripts/reports/SimilarityCsv.sh +++ b/scripts/reports/SimilarityCsv.sh @@ -3,6 +3,7 @@ # Looks for similarity using the Graph Data Science Library of Neo4j and creates CSV reports. # It requires an already running Neo4j graph database with already scanned analyzed artifacts. # The reports (csv files) will be written into the sub directory reports/community. +# Note that "scripts/prepareAnalysis.sh" is required to run prior to this script. # Overrideable Constants (defaults also defined in sub scripts) REPORTS_DIRECTORY=${REPORTS_DIRECTORY:-"reports"} @@ -31,20 +32,8 @@ FULL_REPORT_DIRECTORY="${REPORTS_DIRECTORY}/${REPORT_NAME}" mkdir -p "${FULL_REPORT_DIRECTORY}" # Local Constants -PACKAGE_WEIGHTS_CYPHER_DIR="$CYPHER_DIR/Package_Relationship_Weights" -PACKAGE_METRICS_CYPHER_DIR="$CYPHER_DIR/Metrics" SIMILARITY_CYPHER_DIR="$CYPHER_DIR/Similarity" -# Preparation - Add weights to package DEPENDS_ON relationships -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight_property_for_Interface_Dependencies_to_Package_DEPENDS_ON_Relationship.cypher" -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight_property_to_Package_DEPENDS_ON_Relationship.cypher" -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight25PercentInterfaces_to_Package_DEPENDS_ON_relationships.cypher" -execute_cypher_summarized "${PACKAGE_WEIGHTS_CYPHER_DIR}/Add_weight10PercentInterfaces_to_Package_DEPENDS_ON_relationships.cypher" - -# Preparation - Add Package node properties "incomingDependencies" and "outgoingDependencies" -execute_cypher_summarized "${PACKAGE_METRICS_CYPHER_DIR}/Set_Incoming_Package_Dependencies.cypher" -execute_cypher_summarized "${PACKAGE_METRICS_CYPHER_DIR}/Set_Outgoing_Package_Dependencies.cypher" - # Preparation Similarity - Create package dependencies projection execute_cypher "${SIMILARITY_CYPHER_DIR}/Similarity_0_Delete_Projection.cypher" execute_cypher "${SIMILARITY_CYPHER_DIR}/Similarity_0b_Delete_Subgraph_Projection.cypher" diff --git a/scripts/resetAndScan.sh b/scripts/resetAndScan.sh index 49fa1f5f7..c6ed9acf3 100755 --- a/scripts/resetAndScan.sh +++ b/scripts/resetAndScan.sh @@ -5,44 +5,61 @@ # CAUTION: This script deletes all relationships and nodes in the Neo4j Graph Database. # Note: The environment variable NEO4J_INITIAL_PASSWORD is required to login to Neo4j. -JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"1.12.2"} # Version 1.12.2 is the current version (april 2023) +JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.0.3"} # 2.0.3 is the newest version (june 2023) compatible with Neo4j v5, Version 1.12.2 is compatible with Neo4j v4 +JQASSISTANT_CLI_ARTIFACT=${JQASSISTANT_CLI_ARTIFACT:-"jqassistant-commandline-distribution"} # Neo4j v5: "jqassistant-commandline-distribution", Neo4j v4: "jqassistant-commandline-neo4jv3" +JQASSISTANT_CONFIG_TEMPLATE=${JQASSISTANT_CONFIG_TEMPLATE:-"template-neo4jv5-jqassistant.yaml"} # Neo4j v5: "template-neo4jv5-jqassistant.yaml", Neo4j v4: "template-neo4jv4-jqassistant.yaml" NEO4J_EDITION=${NEO4J_EDITION:-"community"} # Choose "community" or "enterprise" -NEO4J_VERSION=${NEO4J_VERSION:-"4.4.20"} # Version 4.4.x is the current long term support (LTS) version (april 2023) +NEO4J_VERSION=${NEO4J_VERSION:-"5.9.0"} NEO4J_BOLT_PORT=${NEO4J_BOLT_PORT:-"7687"} NEO4J_BOLT_URI=${NEO4J_BOLT_URI:-"bolt://localhost:${NEO4J_BOLT_PORT}"} NEO4J_USER=${NEO4J_USER:-"neo4j"} ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"} # Directory with the Java artifacts to scan and analyze TOOLS_DIRECTORY=${TOOLS_DIRECTORY:-"tools"} # Get the tools directory (defaults to "tools") +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} +echo "resetAndScan: SCRIPTS_DIR=${SCRIPTS_DIR}" + # Internal constants -JQASSISTANT_BIN="${TOOLS_DIRECTORY}/jqassistant-commandline-neo4jv3-${JQASSISTANT_CLI_VERSION}/bin" -JQASSISTANT_NEO4J_OPTIONS="-storeUri ${NEO4J_BOLT_URI} -storeUsername ${NEO4J_USER} -storePassword ${NEO4J_INITIAL_PASSWORD}" +JQASSISTANT_DIRECTORY="${TOOLS_DIRECTORY}/${JQASSISTANT_CLI_ARTIFACT}-${JQASSISTANT_CLI_VERSION}" +JQASSISTANT_BIN="${JQASSISTANT_DIRECTORY}/bin" +#JQASSISTANT_NEO4J_OPTIONS="-D jqassistant.store.uri=${NEO4J_BOLT_URI} -D jqassistant.store.remote.username=${NEO4J_USER} -D jqassistant.store.remote.password=${NEO4J_INITIAL_PASSWORD}" +#JQASSISTANT_NEO4J_OPTIONS=-configurationLocations "${JQASSISTANT_CONFIG}/.jqassistant.yaml" # Check if environment variable is set if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then - echo "Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD= if neccessary" + echo "setupJQAssistant: Creating tools directory <${TOOLS_DIRECTORY}> if neccessary" mkdir -p "${TOOLS_DIRECTORY}" fi # Check if SHARED_DOWNLOADS_DIRECTORY variable is set if [ -z "${SHARED_DOWNLOADS_DIRECTORY}" ]; then - echo "Requires variable SHARED_DOWNLOADS_DIRECTORY to be set. If it is the current directory, then use a dot to reflect that." + echo "setupJQAssistant: Error: Requires variable SHARED_DOWNLOADS_DIRECTORY to be set. If it is the current directory, then use a dot to reflect that." exit 1 else # Create shared downloads directory if it doesn't exists - echo "Creating shared downloads directory <${SHARED_DOWNLOADS_DIRECTORY}> if neccessary" + echo "setupJQAssistant: Creating shared downloads directory <${SHARED_DOWNLOADS_DIRECTORY}> if neccessary" mkdir -p "${SHARED_DOWNLOADS_DIRECTORY}" fi # Internal constants -JQASSISTANT_INSTALLATION_NAME="jqassistant-commandline-neo4jv3-${JQASSISTANT_CLI_VERSION}" +JQASSISTANT_INSTALLATION_NAME="${JQASSISTANT_CLI_ARTIFACT}-${JQASSISTANT_CLI_VERSION}" JQASSISTANT_INSTALLATION_DIRECTORY="${TOOLS_DIRECTORY}/${JQASSISTANT_INSTALLATION_NAME}" # Download and unpack jQAssistant @@ -38,17 +41,25 @@ if [ ! -d "${JQASSISTANT_INSTALLATION_DIRECTORY}" ] ; then # Download jQAssistant if [ ! -f "${SHARED_DOWNLOADS_DIRECTORY}/${JQASSISTANT_INSTALLATION_NAME}.zip" ] ; then - echo "Downloading ${JQASSISTANT_INSTALLATION_NAME}.zip" - + jqassistant_cli_fulldownload_url=${JQASSISTANT_CLI_DOWNLOAD_URL}/${JQASSISTANT_CLI_ARTIFACT}/${JQASSISTANT_CLI_VERSION}/${JQASSISTANT_CLI_ARTIFACT}-${JQASSISTANT_CLI_VERSION}-${JQASSISTANT_CLI_DISTRIBUTION} + echo "setupJQAssistant: Downloading ${JQASSISTANT_INSTALLATION_NAME}.zip from ${jqassistant_cli_fulldownload_url}" + # Download jQAssistant # With the option "-L" a redirection will be followed automatically - curl -L --fail-with-body -o "${SHARED_DOWNLOADS_DIRECTORY}/${JQASSISTANT_INSTALLATION_NAME}.zip" https://repo1.maven.org/maven2/com/buschmais/jqassistant/cli/jqassistant-commandline-neo4jv3/$JQASSISTANT_CLI_VERSION/jqassistant-commandline-neo4jv3-$JQASSISTANT_CLI_VERSION-distribution.zip || exit 1 + curl -L --fail-with-body -o "${SHARED_DOWNLOADS_DIRECTORY}/${JQASSISTANT_INSTALLATION_NAME}.zip" "${jqassistant_cli_fulldownload_url}" else - echo "${JQASSISTANT_INSTALLATION_NAME} already downloaded" + echo "setupJQAssistant: ${JQASSISTANT_INSTALLATION_NAME} already downloaded" + fi + + downloaded_file_size=$(wc -c "${SHARED_DOWNLOADS_DIRECTORY}/${JQASSISTANT_INSTALLATION_NAME}.zip" | awk '{print $1}') + if [[ "${downloaded_file_size}" -le 1000 ]]; then + echo "setupJQAssistant: Error: Failed to download ${JQASSISTANT_INSTALLATION_NAME}. Invalid Filesize" + rm -f "${SHARED_DOWNLOADS_DIRECTORY}/${JQASSISTANT_INSTALLATION_NAME}.zip" + exit 1 fi # Unpack the ZIP file (-q option for less verbose output) unzip -q "${SHARED_DOWNLOADS_DIRECTORY}/${JQASSISTANT_INSTALLATION_NAME}.zip" -d "${TOOLS_DIRECTORY}" else - echo "${JQASSISTANT_INSTALLATION_NAME}.zip already installed" + echo "setupJQAssistant: ${JQASSISTANT_INSTALLATION_NAME}.zip already installed" fi \ No newline at end of file diff --git a/scripts/setupNeo4j.sh b/scripts/setupNeo4j.sh index 2939d0023..e808696df 100755 --- a/scripts/setupNeo4j.sh +++ b/scripts/setupNeo4j.sh @@ -5,9 +5,11 @@ # Note: The environment variable NEO4J_INITIAL_PASSWORD needs to be set. NEO4J_EDITION=${NEO4J_EDITION:-"community"} # Choose "community" or "enterprise" -NEO4J_VERSION=${NEO4J_VERSION:-"4.4.20"} # Version 4.4.x is the current long term support (LTS) version (april 2023) -NEO4J_APOC_PLUGIN_VERSION=${NEO4J_APOC_PLUGIN_VERSION:-"4.4.0.15"} #Awesome Procedures for Neo4j Plugin Version 4.4.0.x of is compatible with Neo4j 4.4.x -NEO4J_GDS_PLUGIN_VERSION=${NEO4J_GDS_PLUGIN_VERSION:-"2.3.4"} # Graph Data Science Plugin Version 2.3.x of is compatible with Neo4j 4.4.x +NEO4J_VERSION=${NEO4J_VERSION:-"5.9.0"} +NEO4J_APOC_PLUGIN_VERSION=${NEO4J_APOC_PLUGIN_VERSION:-"5.9.0"} #Awesome Procedures for Neo4j Plugin, Version needs to be compatible to Neo4j +NEO4J_APOC_PLUGIN_EDITION=${NEO4J_APOC_PLUGIN_EDITION:-"core"} #Awesome Procedures for Neo4j Plugin Edition (Neo4j v4.4.x "all", Neo4j >= v5 "core") +NEO4J_APOC_PLUGIN_GITHUB=${NEO4J_APOC_PLUGIN_GITHUB:-"neo4j/apoc"} #Awesome Procedures for Neo4j Plugin GitHub User/Repository (Neo4j v4.4.x "neo4j-contrib/neo4j-apoc-procedures", Neo4j >= v5 "neo4j/apoc") +NEO4J_GDS_PLUGIN_VERSION=${NEO4J_GDS_PLUGIN_VERSION:-"2.4.0"} # Graph Data Science Plugin Version 2.4.x of is compatible with Neo4j 5.x NEO4J_DATA_PATH=${NEO4J_DATA_PATH:-"$( pwd -P )/data"} # Path where Neo4j writes its data to (outside tools dir) NEO4J_RUNTIME_PATH=${NEO4J_RUNTIME_PATH:-"$( pwd -P )/runtime"} # Path where Neo4j puts runtime data to (e.g. logs) (outside tools dir) TOOLS_DIRECTORY=${TOOLS_DIRECTORY:-"tools"} # Get the tools directory (defaults to "tools") @@ -22,7 +24,7 @@ NEO4J_INSTALLATION_NAME="neo4j-${NEO4J_EDITION}-${NEO4J_VERSION}" NEO4J_INSTALLATION_DIRECTORY="${TOOLS_DIRECTORY}/${NEO4J_INSTALLATION_NAME}" NEO4J_CONFIG="${NEO4J_INSTALLATION_DIRECTORY}/conf/neo4j.conf" NEO4J_APOC_CONFIG="${NEO4J_INSTALLATION_DIRECTORY}/conf/apoc.conf" -NEO4J_APOC_PLUGIN_ARTIFACT="apoc-${NEO4J_APOC_PLUGIN_VERSION}-all.jar" +NEO4J_APOC_PLUGIN_ARTIFACT="apoc-${NEO4J_APOC_PLUGIN_VERSION}-${NEO4J_APOC_PLUGIN_EDITION}.jar" NEO4J_GDS_PLUGIN_ARTIFACT="neo4j-graph-data-science-${NEO4J_GDS_PLUGIN_VERSION}.jar" ## Get this "scripts" directory if not already set @@ -34,7 +36,7 @@ echo "setupNeo4j: SCRIPTS_DIR=$SCRIPTS_DIR" # Check if TOOLS_DIRECTORY variable is set if [ -z "${TOOLS_DIRECTORY}" ]; then - echo "setupNeo4j: Requires variable TOOLS_DIRECTORY to be set. If it is the current directory, then use a dot to reflect that." + echo "setupNeo4j: Error: Requires variable TOOLS_DIRECTORY to be set. If it is the current directory, then use a dot to reflect that." exit 1 else # Create tools directory if it doesn't exists @@ -44,7 +46,7 @@ fi # Check if SHARED_DOWNLOADS_DIRECTORY variable is set if [ -z "${SHARED_DOWNLOADS_DIRECTORY}" ]; then - echo "setupNeo4j: Requires variable SHARED_DOWNLOADS_DIRECTORY to be set. If it is the current directory, then use a dot to reflect that." + echo "setupNeo4j: Error: Requires variable SHARED_DOWNLOADS_DIRECTORY to be set. If it is the current directory, then use a dot to reflect that." exit 1 else # Create shared downloads directory if it doesn't exists @@ -52,6 +54,12 @@ else mkdir -p "${SHARED_DOWNLOADS_DIRECTORY}" fi +# Check if environment variable is set +if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then + echo "setupNeo4j: Error: Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD=> "${NEO4J_CONFIG}" + if [[ "$NEO4J_MAJOR_VERSION_NUMBER" -ge 5 ]]; then + echo "setupNeo4j: Neo4j v5 or higher detected" + { + echo "" + echo "# Paths of data directories in the installation (v5)" + echo "server.directories.data=${NEO4J_DATA_PATH}" + echo "server.directories.logs=${NEO4J_RUNTIME_PATH}/logs" + echo "server.directories.dumps.root=${NEO4J_RUNTIME_PATH}/dumps" + echo "server.directories.run=${NEO4J_RUNTIME_PATH}/run" + echo "server.directories.transaction.logs.root=${NEO4J_DATA_PATH}/transactions" + echo "" + echo "# Ports Configuration (v5)" + echo "server.bolt.listen_address=:${NEO4J_BOLT_PORT}" + echo "server.bolt.advertised_address=:${NEO4J_BOLT_PORT}" + echo "server.http.listen_address=:${NEO4J_HTTP_PORT}" + echo "server.http.advertised_address=:${NEO4J_HTTP_PORT}" + echo "server.https.listen_address=:${NEO4J_HTTPS_PORT}" + echo "server.https.advertised_address=:${NEO4J_HTTPS_PORT}" + } >> "${NEO4J_CONFIG}" + else + echo "setupNeo4j: Neo4j v4 or lower detected" + { + echo "" + echo "# Paths of data directories in the installation" + echo "dbms.directories.data=${NEO4J_DATA_PATH}" + echo "dbms.directories.logs=${NEO4J_RUNTIME_PATH}/logs" + echo "dbms.directories.dumps.root=${NEO4J_RUNTIME_PATH}/dumps" + echo "dbms.directories.run=${NEO4J_RUNTIME_PATH}/run" + echo "dbms.directories.transaction.logs.root=${NEO4J_DATA_PATH}/transactions" + echo "" + echo "# Ports Configuration" + echo "dbms.connector.bolt.listen_address=:${NEO4J_BOLT_PORT}" + echo "dbms.connector.bolt.advertised_address=:${NEO4J_BOLT_PORT}" + echo "dbms.connector.http.listen_address=:${NEO4J_HTTP_PORT}" + echo "dbms.connector.http.advertised_address=:${NEO4J_HTTP_PORT}" + echo "dbms.connector.https.listen_address=:${NEO4J_HTTPS_PORT}" + echo "dbms.connector.https.advertised_address=:${NEO4J_HTTPS_PORT}" + } >> "${NEO4J_CONFIG}" + fi echo "setupNeo4j: Configuring static settings (memory, procedure permittions, ...)" - cat "${SCRIPTS_DIR}/templates/template-neo4j.conf" >> "${NEO4J_CONFIG}" + if [[ "$NEO4J_MAJOR_VERSION_NUMBER" -ge 5 ]]; then + cat "${SCRIPTS_DIR}/templates/template-neo4j.conf" >> "${NEO4J_CONFIG}" + else + cat "${SCRIPTS_DIR}/templates/template-neo4j-v4.conf" >> "${NEO4J_CONFIG}" + fi # Set initial password for user "neo4j" otherwise the default password "neo4j" would need to be changed immediately (prompt). # This needs to be done after the configuration changes. @@ -127,15 +159,21 @@ if [ ! -f "${NEO4J_INSTALLATION_DIRECTORY}/plugins/${NEO4J_APOC_PLUGIN_ARTIFACT} # Download the Neo4j Plugin "Awesome Procedures for Neo4j" if [ ! -f "${SHARED_DOWNLOADS_DIRECTORY}/${NEO4J_APOC_PLUGIN_ARTIFACT}" ] ; then - echo "setupNeo4j: Downloading ${NEO4J_APOC_PLUGIN_ARTIFACT}" - # Download the Neo4j Plugin "Awesome Procedures for Neo4j" echo "setupNeo4j: Downloading ${NEO4J_APOC_PLUGIN_ARTIFACT}" - curl -L --fail-with-body -o "${SHARED_DOWNLOADS_DIRECTORY}/${NEO4J_APOC_PLUGIN_ARTIFACT}" https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/${NEO4J_APOC_PLUGIN_VERSION}/apoc-${NEO4J_APOC_PLUGIN_VERSION}-all.jar || exit 1 + curl -L --fail-with-body -o "${SHARED_DOWNLOADS_DIRECTORY}/${NEO4J_APOC_PLUGIN_ARTIFACT}" https://github.com/${NEO4J_APOC_PLUGIN_GITHUB}/releases/download/${NEO4J_APOC_PLUGIN_VERSION}/apoc-${NEO4J_APOC_PLUGIN_VERSION}-${NEO4J_APOC_PLUGIN_EDITION}.jar || exit 1 else echo "setupNeo4j: ${NEO4J_APOC_PLUGIN_ARTIFACT} already downloaded" fi + # Check downloaded file size to be at least 100 bytes + downloaded_file_size=$(wc -c "${SHARED_DOWNLOADS_DIRECTORY}/${NEO4J_APOC_PLUGIN_ARTIFACT}") + if [[ "$downloaded_file_size" -le 100 ]]; then + echo "setupNeo4j: Error: Failed to download ${NEO4J_APOC_PLUGIN_ARTIFACT}: Invalid Filesize." + rm -f "${SHARED_DOWNLOADS_DIRECTORY}/${NEO4J_APOC_PLUGIN_ARTIFACT}" + exit 1 + fi + # Uninstall previously installed Neo4j Plugin "Awesome Procedures for Neo4j" (APOC) rm -f "${NEO4J_INSTALLATION_DIRECTORY}/plugins/apoc*.jar" @@ -145,7 +183,7 @@ if [ ! -f "${NEO4J_INSTALLATION_DIRECTORY}/plugins/${NEO4J_APOC_PLUGIN_ARTIFACT} # Fail if Neo4j Plugin "Awesome Procedures for Neo4j" (APOC) hadn't been downloaded successfully if [ ! -f "${NEO4J_INSTALLATION_DIRECTORY}/plugins/${NEO4J_APOC_PLUGIN_ARTIFACT}" ] ; then - echo "setupNeo4j: Failed to download and install ${NEO4J_APOC_PLUGIN_ARTIFACT}" + echo "setupNeo4j: Error: Failed to download and install ${NEO4J_APOC_PLUGIN_ARTIFACT}" exit 1 fi @@ -172,6 +210,14 @@ if [ ! -f "${NEO4J_INSTALLATION_DIRECTORY}/plugins/${NEO4J_GDS_PLUGIN_ARTIFACT}" echo "setupNeo4j: ${NEO4J_GDS_PLUGIN_ARTIFACT} already downloaded" fi + # Check downloaded file size to be at least 100 bytes + downloaded_file_size=$(wc -c "${SHARED_DOWNLOADS_DIRECTORY}/${NEO4J_GDS_PLUGIN_ARTIFACT}" | awk '{print $1}') + if [[ "$downloaded_file_size" -le 100 ]]; then + echo "setupNeo4j: Error: Failed to download ${NEO4J_GDS_PLUGIN_ARTIFACT}. Invalid Filesize." + rm -f "${SHARED_DOWNLOADS_DIRECTORY}/${NEO4J_GDS_PLUGIN_ARTIFACT}" + exit 1 + fi + # Uninstall previously installed Neo4j Plugin "Graph Data Science" (GDS) rm -f "${NEO4J_INSTALLATION_DIRECTORY}/plugins/neo4j-graph-data-science*.jar" @@ -181,7 +227,7 @@ if [ ! -f "${NEO4J_INSTALLATION_DIRECTORY}/plugins/${NEO4J_GDS_PLUGIN_ARTIFACT}" # Fail if Neo4j Plugin "Graph Data Science" (GDS) hadn't been downloaded successfully if [ ! -f "${NEO4J_INSTALLATION_DIRECTORY}/plugins/${NEO4J_GDS_PLUGIN_ARTIFACT}" ] ; then - echo "setupNeo4j: Failed to download and install ${NEO4J_GDS_PLUGIN_ARTIFACT}" + echo "setupNeo4j: Error: Failed to download and install ${NEO4J_GDS_PLUGIN_ARTIFACT}" exit 1 fi else diff --git a/scripts/setupNeo4jInitialPassword.sh b/scripts/setupNeo4jInitialPassword.sh index e60c42b20..6944f50e2 100755 --- a/scripts/setupNeo4jInitialPassword.sh +++ b/scripts/setupNeo4jInitialPassword.sh @@ -5,7 +5,7 @@ # Note: The environment variable NEO4J_INITIAL_PASSWORD needs to be set. NEO4J_EDITION=${NEO4J_EDITION:-"community"} # Choose "community" or "enterprise" -NEO4J_VERSION=${NEO4J_VERSION:-"4.4.20"} # Version 4.4.x is the current long term support (LTS) version (april 2023) +NEO4J_VERSION=${NEO4J_VERSION:-"5.9.0"} TOOLS_DIRECTORY=${TOOLS_DIRECTORY:-"tools"} # Get the tools directory (defaults to "tools") # Internal constants @@ -24,6 +24,22 @@ if [ ! -d "${NEO4J_INSTALLATION_DIRECTORY}" ] ; then exit 1 fi -# Set the initial password using a temporary NEO4J_HOME environment variable pointing to the current setup +# Print the environment variable NEO4J_HOME that is used to execute the command echo "setupNeo4jInitialPassword: Using ${NEO4J_INSTALLATION_DIRECTORY} as NEO4J_HOME" -NEO4J_HOME="${NEO4J_INSTALLATION_DIRECTORY}" ${NEO4J_INSTALLATION_DIRECTORY}/bin/neo4j-admin set-initial-password "${NEO4J_INITIAL_PASSWORD}" \ No newline at end of file + +# Extract the first component of the version number (=major version number) +NEO4J_MAJOR_VERSION_NUMBER=$(echo "$NEO4J_VERSION" | cut -d'.' -f1) + +# Check if the first component is greater than or equal to 5 +if [[ "$NEO4J_MAJOR_VERSION_NUMBER" -ge 5 ]]; then + echo "setupNeo4jInitialPassword: Neo4j v5 or higher detected" + # Neo4j version < 5 + # Set the initial password using a temporary NEO4J_HOME environment variable pointing to the current setup + NEO4J_HOME="${NEO4J_INSTALLATION_DIRECTORY}" ${NEO4J_INSTALLATION_DIRECTORY}/bin/neo4j-admin dbms set-initial-password "${NEO4J_INITIAL_PASSWORD}" +else + echo "setupNeo4jInitialPassword: Neo4j v4 or lower detected" + # Neo4j version >= 5 + # Set the initial password using a temporary NEO4J_HOME environment variable pointing to the current setup + NEO4J_HOME="${NEO4J_INSTALLATION_DIRECTORY}" ${NEO4J_INSTALLATION_DIRECTORY}/bin/neo4j-admin set-initial-password "${NEO4J_INITIAL_PASSWORD}" +fi + diff --git a/scripts/startNeo4j.sh b/scripts/startNeo4j.sh index 4e4b7b93a..469d508a4 100755 --- a/scripts/startNeo4j.sh +++ b/scripts/startNeo4j.sh @@ -6,14 +6,14 @@ # Note: It requires Neo4j to be installed in the TOOLS_DIRECTORY. NEO4J_EDITION=${NEO4J_EDITION:-"community"} # Choose "community" or "enterprise" -NEO4J_VERSION=${NEO4J_VERSION:-"4.4.20"} # Version 4.4.x is the current long term support (LTS) version (april 2023) +NEO4J_VERSION=${NEO4J_VERSION:-"5.9.0"} TOOLS_DIRECTORY=${TOOLS_DIRECTORY:-"tools"} # Get the tools directory (defaults to "tools") NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} # Internal Constants NEO4J_DIR="${TOOLS_DIRECTORY}/neo4j-${NEO4J_EDITION}-${NEO4J_VERSION}" NEO4J_BIN="${NEO4J_DIR}/bin" -WAIT_TIMES="1 2 4 8 16" +WAIT_TIMES="1 2 4 8 16 32" ## Get this "scripts" directory if not already set # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. @@ -37,7 +37,7 @@ else fi # Check if Neo4j is stopped (not running) using a temporary NEO4J_HOME environment variable that points to the current installation -if NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status | grep -q "not" ; then +if NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status 2>&1 | grep -q "not running" ; then echo "startNeo4j: Starting neo4j-${NEO4J_EDITION}-${NEO4J_VERSION} in ${NEO4J_DIR}" # Check if there is already a process that listens to the Neo4j HTTP port @@ -57,7 +57,7 @@ if NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status | grep -q "not" ; then echo "startNeo4j: Waiting for ${waitTime} second(s)" sleep ${waitTime} - if ! NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status | grep -q "not" ; then + if ! NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status 2>&1 | grep -q "not running" ; then echo "startNeo4j: Successfully started neo4j-${NEO4J_EDITION}-${NEO4J_VERSION}" exit 0 fi @@ -67,7 +67,7 @@ else fi # Check if Neo4j is still not running using a temporary NEO4J_HOME environment variable that points to the current installation -if NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status | grep -q "not" ; then +if NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status 2>&1 | grep -q "not running" ; then echo "startNeo4j: neo4j-${NEO4J_EDITION}-${NEO4J_VERSION} still not running. Something went wrong. Details see 'NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status'." exit 1 fi diff --git a/scripts/stopNeo4j.sh b/scripts/stopNeo4j.sh index 91d36c3c8..d81079c3d 100755 --- a/scripts/stopNeo4j.sh +++ b/scripts/stopNeo4j.sh @@ -5,7 +5,7 @@ # Note: Does nothing if the database is already stopped. NEO4J_EDITION=${NEO4J_EDITION:-"community"} # Choose "community" or "enterprise" -NEO4J_VERSION=${NEO4J_VERSION:-"4.4.20"} # Version 4.4.x is the current long term support (LTS) version (april 2023) +NEO4J_VERSION=${NEO4J_VERSION:-"5.9.0"} TOOLS_DIRECTORY=${TOOLS_DIRECTORY:-"tools"} # Get the tools directory (defaults to "tools") NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} @@ -28,7 +28,7 @@ else fi # Check if Neo4j is stopped (not running) using a temporary NEO4J_HOME environment variable that points to the current installation -if NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status | grep -q "not" ; then +if NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status 2>&1 | grep -q "not running" ; then echo "stopNeo4j: neo4j-${NEO4J_EDITION}-${NEO4J_VERSION} aleady stopped" exit 0 else @@ -37,7 +37,7 @@ else fi # Check if Neo4j is still not running using a temporary NEO4J_HOME environment variable that points to the current installation -if NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status | grep -q "not" ; then +if NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status 2>&1 | grep -q "not running" ; then echo "stopNeo4j: Successfully stopped neo4j-${NEO4J_EDITION}-${NEO4J_VERSION}" else echo "stopNeo4j: neo4j-${NEO4J_EDITION}-${NEO4J_VERSION} still running. Something went wrong. Details see 'NEO4J_HOME=${NEO4J_DIR} ${NEO4J_BIN}/neo4j status'." diff --git a/scripts/templates/template-neo4j-v4.conf b/scripts/templates/template-neo4j-v4.conf new file mode 100644 index 000000000..ce0b02661 --- /dev/null +++ b/scripts/templates/template-neo4j-v4.conf @@ -0,0 +1,22 @@ + +# The following static configuration entries were taken from "template-neo4j.conf". + +# List of procedures and user defined functions that are allowed +# full access to the database through unsupported/insecure internal APIs. +dbms.security.procedures.unrestricted=apoc.*,gds.* + +# Memory: Java Heap Size +dbms.memory.heap.initial_size=1024m +dbms.memory.heap.max_size=1024m + +# Memory: The amount of memory to use for mapping the store files. +dbms.memory.pagecache.size=16m + +# Memory: Exits JVM on the first occurrence of an out-of-memory error. +dbms.jvm.additional=-XX:+ExitOnOutOfMemoryError + +# Memory: Limit the amount of memory that all of the running transaction can consume. +dbms.memory.transaction.global_max_size=384m + +# Memory: Limit the amount of memory that a single transaction can consume. +dbms.memory.transaction.max_size=256m \ No newline at end of file diff --git a/scripts/templates/template-neo4j.conf b/scripts/templates/template-neo4j.conf index 679814880..fb7a764e4 100644 --- a/scripts/templates/template-neo4j.conf +++ b/scripts/templates/template-neo4j.conf @@ -6,23 +6,17 @@ dbms.security.procedures.unrestricted=apoc.*,gds.* # Memory: Java Heap Size -dbms.memory.heap.initial_size=1024m -dbms.memory.heap.max_size=1024m server.memory.heap.initial_size=1024m server.memory.heap.max_size=1024m # Memory: The amount of memory to use for mapping the store files. -dbms.memory.pagecache.size=16m server.memory.pagecache.size=16m # Memory: Exits JVM on the first occurrence of an out-of-memory error. -dbms.jvm.additional=-XX:+ExitOnOutOfMemoryError server.jvm.additional=-XX:+ExitOnOutOfMemoryError # Memory: Limit the amount of memory that all of the running transaction can consume. -dbms.memory.transaction.global_max_size=384m db.memory.transaction.total.max=384m # Memory: Limit the amount of memory that a single transaction can consume. -dbms.memory.transaction.max_size=256m db.memory.transaction.max=256m \ No newline at end of file diff --git a/scripts/templates/template-neo4jv4-jqassistant.yaml b/scripts/templates/template-neo4jv4-jqassistant.yaml new file mode 100644 index 000000000..23dffb513 --- /dev/null +++ b/scripts/templates/template-neo4jv4-jqassistant.yaml @@ -0,0 +1,215 @@ +jqassistant: + # Controls whether the execution of jQAssistant shall be skipped + # + # -Djqassistant.skip: true|false + skip: false + + # Defines the local and remote Maven repositories for retrieving additional plugins. + repositories: + # The path to the local repository. + # + # -Djqassistant.repositories.local (optional) + local: ./../downloads/jqassistant-repository + + # The list of plugins to load and activate. + # + # Each plugin is identified using its Maven coordinates: + # + # -Djqassistant.plugins[0].group-id + # -Djqassistant.plugins[0].artifact-id + # -Djqassistant.plugins[0].version + # -Djqassistant.plugins[0].classifier (optional) + # -Djqassistant.plugins[0].type (optional) + #plugins: + # - group-id: + # artifact-id: + # version: + # classifier: + # type: + + # The store configuration + store: + # URI of the database to connect to. Supported URI schemes are 'file' for embedded databases and 'bolt' for connecting to a running Neo4j instance (3.x+), e.g. + # + # -Djqassistant.store.uri + #uri: file://target/jqassistant/store + uri: bolt://localhost:${NEO4J_BOLT_PORT:7687} + + # Settings for the embedded Neo4j store + embedded: + # Enable the HTTP and BOLT connector for the embedded store + # + # -Djqassistant.store.embedded.connector-enabled: true|false + connector-enabled: false + + # The listen address for the HTTP and BOLT connectors + # + # -Djqassistant.store.embedded.listen-address + listen-address: localhost + + # The BOLT connector port + # + # -Djqassistant.store.embedded.bolt-port + bolt-port: 7687 + + # The HTTP connector port + # + # -Djqassistant.store.embedded.http-port: 7474 + http-port: 7474 + + # Settings for connecting to a remote Neo4j store + remote: + # The user name for authentication. + # + # -Djqassistant.store.username + username: ${NEO4J_USER:neo4j} + + # The password for authentication. + # + # -Djqassistant.store.password + password: ${NEO4J_INITIAL_PASSWORD} + + # Activate encryption level for 'bolt' connections. + # + # -Djqassistant.store.encryption: true|false + encryption: false + + #The trust strategy for 'bolt' connections + # + # -Djqassistant.store.trust-strategy: trustAllCertificates|trustCustomCaSignedCertificates|trustSystemCaSignedCertificates + trust-strategy: trustAllCertificates + + # The file containing the custom CA certificate for trust strategy. + # + # -Djqassistant.store.trust-certificate + trust-certificate: + + # Additional properties to be passed to the remote store as key-value pairs. + # + # -Djqassistant.store.properties + properties: + + # The Scanner configuration + scan: + # Indicates whether to initially reset the store (i.e. clear all nodes and relationships) before scanning. + # + # -Djqassistant.scan.reset: true|false + reset: true + + # Specifies if a scan shall be continued if an error is encountered. + # + # -Djqassistant.scan.continue-on-error: true|false + continue-on-error: false + + # The items to include for scanning. + include: + # A list of files to include. + # + #jqassistant.scan.include.files[0] + files: + # - src/folder + + # A list of URLs to include. + # + #jqassistant.scan.include.urls[0] + urls: + # - maven:repository::https://nexus/repository + + # The properties to configure scanner plugins as key-value pairs. The supported properties are plugin specific. + # + # -Djqassistant.scan.properties + properties: + # plugin.property.key: value + + # The analyze configuration + analyze: + # The rule configuration + rule: + # The name of the directory containing project rules. + # + # -Djqassistant.analyze.rule.directory + directory: jqassistant/rules + + # The default severity of concepts without an explicit severity. + # + # -Djqassistant.analyze.rule.default-concept-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER + default-concept-severity: MINOR + + # The default severity of constraint without an explicit severity. + # + # -Djqassistant.analyze.rule.default-constraint-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER + default-constraint-severity: MAJOR + + # The default severity of groups without an explicit severity. + # + # -Djqassistant.analyze.rule.default-group-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER + default-group-severity: + + # The report configuration + report: + # The properties to configure report plugins. The supported properties are plugin specific. + # + # -Djqassistant.analyze.report.properties + properties: + # plugin.property.key: value + + # Determines the severity level for reporting a warning. + # + # -Djqassistant.analyze.report.warn-on-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER|NEVER + warn-on-severity: MINOR + + # Determines the severity level for reporting a failure. + # + # -Djqassistant.analyze.report.fail-on-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER|NEVER + fail-on-severity: MAJOR + + # Determines if jQAssistant shall continue the build if failures have been detected. + # + # -Djqassistant.analyze.report.continue-on-failure: true|false + continue-on-failure: false + + # Create an archive containing all generated reports. + # + # -Djqassistant.analyze.report.create-archive: true|false + create-archive: false + + # Output directory for analysis reports + directory: ./runtime/jqassistant/report + + # The concepts to be applied. + # + # -Djqassistant.analyze.concepts[0] + # Migration to v2: https://github.com/jQAssistant/jqa-java-plugin/blob/master/src/main/asciidoc/release-notes.adoc + concepts: + - classpath:Resolve + - dependency:Package + - dependency:Artifact + - java:JavaVersion + + # The constraints to be validated. + # + # -Djqassistant.analyze.constraints[0] + constraints: + # - my-constraint + + # The groups to be executed. + # + # -Djqassistant.analyze.groups[0] + groups: + # - spring-boot:Default + + # The parameters to be passed to the executed rules. + # + # -Djqassistant.analyze.rule-parameters."parameterName" + rule-parameters: + # parameterName: value + + # Execute concepts even if they have already been applied before + # + # -Djqassistant.analyze.execute-applied-concepts: true|false + execute-applied-concepts: false + + # The execution time [seconds] for rules (concepts/constraints) to show a warning. Can be used as a hint for optimization. + # + # -Djqassistant.analyze.warn-on-rule-execution-time-seconds + warn-on-rule-execution-time-seconds: 5 diff --git a/scripts/templates/template-neo4jv5-jqassistant.yaml b/scripts/templates/template-neo4jv5-jqassistant.yaml new file mode 100644 index 000000000..d64939b62 --- /dev/null +++ b/scripts/templates/template-neo4jv5-jqassistant.yaml @@ -0,0 +1,215 @@ +jqassistant: + # Controls whether the execution of jQAssistant shall be skipped + # + # -Djqassistant.skip: true|false + skip: false + + # Defines the local and remote Maven repositories for retrieving additional plugins. + repositories: + # The path to the local repository. + # + # -Djqassistant.repositories.local (optional) + local: ./../downloads/jqassistant-repository + + # The list of plugins to load and activate. + # + # Each plugin is identified using its Maven coordinates: + # + # -Djqassistant.plugins[0].group-id + # -Djqassistant.plugins[0].artifact-id + # -Djqassistant.plugins[0].version + # -Djqassistant.plugins[0].classifier (optional) + # -Djqassistant.plugins[0].type (optional) + #plugins: + # - group-id: + # artifact-id: + # version: + # classifier: + # type: + + # The store configuration + store: + # URI of the database to connect to. Supported URI schemes are 'file' for embedded databases and 'bolt' for connecting to a running Neo4j instance (3.x+), e.g. + # + # -Djqassistant.store.uri + #uri: file://target/jqassistant/store + uri: bolt://localhost:${NEO4J_BOLT_PORT:7687} + + # Settings for the embedded Neo4j store + embedded: + # Enable the HTTP and BOLT connector for the embedded store + # + # -Djqassistant.store.embedded.connector-enabled: true|false + connector-enabled: false + + # The listen address for the HTTP and BOLT connectors + # + # -Djqassistant.store.embedded.listen-address + listen-address: localhost + + # The BOLT connector port + # + # -Djqassistant.store.embedded.bolt-port + bolt-port: 7687 + + # The HTTP connector port + # + # -Djqassistant.store.embedded.http-port: 7474 + http-port: 7474 + + # Settings for connecting to a remote Neo4j store + remote: + # The user name for authentication. + # + # -Djqassistant.store.username + username: ${NEO4J_USER:neo4j} + + # The password for authentication. + # + # -Djqassistant.store.password + password: ${NEO4J_INITIAL_PASSWORD} + + # Activate encryption level for 'bolt' connections. + # + # -Djqassistant.store.encryption: true|false + encryption: false + + #The trust strategy for 'bolt' connections + # + # -Djqassistant.store.trust-strategy: trustAllCertificates|trustCustomCaSignedCertificates|trustSystemCaSignedCertificates + trust-strategy: trustAllCertificates + + # The file containing the custom CA certificate for trust strategy. + # + # -Djqassistant.store.trust-certificate + trust-certificate: + + # Additional properties to be passed to the remote store as key-value pairs. + # + # -Djqassistant.store.properties + properties: + + # The Scanner configuration + scan: + # Indicates whether to initially reset the store (i.e. clear all nodes and relationships) before scanning. + # + # -Djqassistant.scan.reset: true|false + reset: true + + # Specifies if a scan shall be continued if an error is encountered. + # + # -Djqassistant.scan.continue-on-error: true|false + continue-on-error: false + + # The items to include for scanning. + include: + # A list of files to include. + # + #jqassistant.scan.include.files[0] + files: + # - src/folder + + # A list of URLs to include. + # + #jqassistant.scan.include.urls[0] + urls: + # - maven:repository::https://nexus/repository + + # The properties to configure scanner plugins as key-value pairs. The supported properties are plugin specific. + # + # -Djqassistant.scan.properties + properties: + # plugin.property.key: value + + # The analyze configuration + analyze: + # The rule configuration + rule: + # The name of the directory containing project rules. + # + # -Djqassistant.analyze.rule.directory + directory: jqassistant/rules + + # The default severity of concepts without an explicit severity. + # + # -Djqassistant.analyze.rule.default-concept-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER + default-concept-severity: MINOR + + # The default severity of constraint without an explicit severity. + # + # -Djqassistant.analyze.rule.default-constraint-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER + default-constraint-severity: MAJOR + + # The default severity of groups without an explicit severity. + # + # -Djqassistant.analyze.rule.default-group-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER + default-group-severity: + + # The report configuration + report: + # The properties to configure report plugins. The supported properties are plugin specific. + # + # -Djqassistant.analyze.report.properties + properties: + # plugin.property.key: value + + # Determines the severity level for reporting a warning. + # + # -Djqassistant.analyze.report.warn-on-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER|NEVER + warn-on-severity: MINOR + + # Determines the severity level for reporting a failure. + # + # -Djqassistant.analyze.report.fail-on-severity: INFO|MINOR|MAJOR|CRITICAL|BLOCKER|NEVER + fail-on-severity: MAJOR + + # Determines if jQAssistant shall continue the build if failures have been detected. + # + # -Djqassistant.analyze.report.continue-on-failure: true|false + continue-on-failure: false + + # Create an archive containing all generated reports. + # + # -Djqassistant.analyze.report.create-archive: true|false + create-archive: false + + # Output directory for analysis reports + directory: ./runtime/jqassistant/report + + # The concepts to be applied. + # + # -Djqassistant.analyze.concepts[0] + # https://github.com/jQAssistant/jqa-java-plugin/blob/e17fc3032fb0d4258d4f5ca0c64099a6c789c070/src/main/asciidoc/release-notes.adoc#L9 + concepts: + - java-classpath:Resolve + - java:PackageDependency + - java:ArtifactDependency + - java:JavaVersion + + # The constraints to be validated. + # + # -Djqassistant.analyze.constraints[0] + constraints: + # - my-constraint + + # The groups to be executed. + # + # -Djqassistant.analyze.groups[0] + groups: + # - spring-boot:Default + + # The parameters to be passed to the executed rules. + # + # -Djqassistant.analyze.rule-parameters."parameterName" + rule-parameters: + # parameterName: value + + # Execute concepts even if they have already been applied before + # + # -Djqassistant.analyze.execute-applied-concepts: true|false + execute-applied-concepts: false + + # The execution time [seconds] for rules (concepts/constraints) to show a warning. Can be used as a hint for optimization. + # + # -Djqassistant.analyze.warn-on-rule-execution-time-seconds + warn-on-rule-execution-time-seconds: 5 diff --git a/scripts/waitForNeo4jHttp.sh b/scripts/waitForNeo4jHttp.sh index bcf1a890c..ddf3df935 100755 --- a/scripts/waitForNeo4jHttp.sh +++ b/scripts/waitForNeo4jHttp.sh @@ -12,35 +12,39 @@ NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} echo "waitForNeo4jHttp: SCRIPTS_DIR=$SCRIPTS_DIR" +# Get the "cypher" directory by taking the path of this script and going one directory up and then to "cypher". +CYPHER_DIR=${CYPHER_DIR:-"${SCRIPTS_DIR}/../cypher"} +echo "waitForNeo4jHttp: CYPHER_DIR=$CYPHER_DIR" + +# Define functions to execute a cypher query from within the given file (first and only argument) +source "${SCRIPTS_DIR}/executeQueryFunctions.sh" + # List of wait times in seconds per retry WAIT_TIMES="1 1 2 4 8 16 32 64" -count_elements_cypher_query="\"MATCH (n)-[r]-(m) RETURN COUNT(DISTINCT n) AS nodeCount, COUNT(DISTINCT r) AS relationshipCount\"" -count_elements_query_request="{\"statements\":[{\"statement\":${count_elements_cypher_query},\"includeStats\": true}]}" - # Wait until the HTTP endpoint is up and a cypher query can be executed + echo "${WAIT_TIMES}" | tr ' ' '\n' | while read -r waitTime; do echo "waitForNeo4jHttp: Waiting for ${waitTime} second(s)" sleep "${waitTime}" # Calls the Neo4j HTTP API using cURL ( https://curl.se ) - if ! cyper_elements_query_result=$(curl --silent -S --fail-with-body -H Accept:application/json -H Content-Type:application/json \ - -u neo4j:"${NEO4J_INITIAL_PASSWORD}" \ - "http://localhost:${NEO4J_HTTP_PORT}/db/data/transaction/commit" \ - -d "${count_elements_query_request}"); + if ! cyper_elements_query_result=$(execute_cypher "${CYPHER_DIR}/Count_nodes_and_relationships.cypher"); then continue; # query failed -> try again fi - cyper_elements_query_error=$( echo "${cyper_elements_query_result}" | jq -r '.errors[0] // empty' ) - if [[ -n "${cyper_elements_query_error}" ]]; then - continue; # query returned an error message -> try again - fi - - cyper_elements_query_success=$( echo "${cyper_elements_query_result}" | jq -r '.results[0] // empty' ) - if [[ -n "${cyper_elements_query_success}" ]]; then - echo "waitForNeo4jHttp: Successfully reached Neo4j HTTP API." - echo -n "${cyper_elements_query_result}" | jq -r '(.results[0])? | .columns,(.data[].row)?|flatten | @csv' + if [[ -n "${cyper_elements_query_result}" ]]; then + echo "waitForNeo4jHttp: Successfully accessed Neo4j HTTP API." + echo "${cyper_elements_query_result}" exit 0 fi done + +if ! cyper_elements_query_result=$(execute_cypher "${CYPHER_DIR}/Count_nodes_and_relationships.cypher"); then + # Error: Couldn't access HTTP after all wait iterations + echo "waitForNeo4jHttp: Error: Failed to access Neo4j HTTP API." + exit 1 +fi + +