From d3a5b96b3e6d52318a6df9174246927e0082975c Mon Sep 17 00:00:00 2001 From: JohT Date: Sat, 1 Jul 2023 15:08:29 +0200 Subject: [PATCH 1/4] Unify message for missing initial Neo4j password --- scripts/executeJupyterNotebook.sh | 2 +- scripts/executeQuery.sh | 2 +- scripts/prepareAnalysis.sh | 2 +- scripts/resetAndScan.sh | 2 +- scripts/setupNeo4j.sh | 2 +- scripts/setupNeo4jInitialPassword.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/executeJupyterNotebook.sh b/scripts/executeJupyterNotebook.sh index 3f3648299..1d2198d24 100755 --- a/scripts/executeJupyterNotebook.sh +++ b/scripts/executeJupyterNotebook.sh @@ -23,7 +23,7 @@ SKIP_JUPYTER_NOTEBOOK_PDF_GENERATION=${SKIP_JUPYTER_NOTEBOOK_PDF_GENERATION:-""} # Check if environment variable is set if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then - echo "executeJupyterNotebook: Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD='." exit 1 fi diff --git a/scripts/executeQuery.sh b/scripts/executeQuery.sh index f27980875..584427de2 100755 --- a/scripts/executeQuery.sh +++ b/scripts/executeQuery.sh @@ -19,7 +19,7 @@ NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/neo4j/tx/ # Check if environment variable is set if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then - echo "Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD='." exit 1 fi diff --git a/scripts/prepareAnalysis.sh b/scripts/prepareAnalysis.sh index e3f31e3f5..5925b65a9 100644 --- a/scripts/prepareAnalysis.sh +++ b/scripts/prepareAnalysis.sh @@ -11,7 +11,7 @@ echo "prepareAnalysis: SCRIPTS_DIR=${SCRIPTS_DIR}" # Check if environment variable is set if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then - echo "prepareAnalysis: Error: Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD='." + echo "prepareAnalysis: Error: Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD='." exit 1 fi diff --git a/scripts/resetAndScan.sh b/scripts/resetAndScan.sh index c6ed9acf3..fbffdf3cb 100755 --- a/scripts/resetAndScan.sh +++ b/scripts/resetAndScan.sh @@ -32,7 +32,7 @@ JQASSISTANT_BIN="${JQASSISTANT_DIRECTORY}/bin" # Check if environment variable is set if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then - echo "resetAndScan: Error: Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD='." exit 1 fi diff --git a/scripts/setupNeo4j.sh b/scripts/setupNeo4j.sh index 1bf64d65f..27173ee5a 100755 --- a/scripts/setupNeo4j.sh +++ b/scripts/setupNeo4j.sh @@ -56,7 +56,7 @@ fi # Check if environment variable is set if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then - echo "setupNeo4j: Error: Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD='." exit 1 fi diff --git a/scripts/setupNeo4jInitialPassword.sh b/scripts/setupNeo4jInitialPassword.sh index 6944f50e2..24b10c611 100755 --- a/scripts/setupNeo4jInitialPassword.sh +++ b/scripts/setupNeo4jInitialPassword.sh @@ -14,7 +14,7 @@ NEO4J_INSTALLATION_DIRECTORY="${TOOLS_DIRECTORY}/${NEO4J_INSTALLATION_NAME}" # Check if environment variable is set if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then - echo "Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD='." exit 1 fi From 7cc1961fe92516dbc53d21dc4d0695db0f53574f Mon Sep 17 00:00:00 2001 From: JohT Date: Sat, 1 Jul 2023 15:18:39 +0200 Subject: [PATCH 2/4] Separate artifacts & their download from analysis --- .github/workflows/code-reports.yml | 11 ++- COMMANDS.md | 62 +++++++++--- README.md | 12 +-- scripts/SCRIPTS.md | 2 +- scripts/analysis/analyze.sh | 97 +++++-------------- .../downloadAxonFramework.sh | 0 scripts/examples/analyzeAxonFramework.sh | 37 +++++++ 7 files changed, 127 insertions(+), 94 deletions(-) rename scripts/{artifacts => downloader}/downloadAxonFramework.sh (100%) create mode 100755 scripts/examples/analyzeAxonFramework.sh diff --git a/.github/workflows/code-reports.yml b/.github/workflows/code-reports.yml index 50c496485..3c8c6990d 100644 --- a/.github/workflows/code-reports.yml +++ b/.github/workflows/code-reports.yml @@ -97,13 +97,22 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/*.sh') }} + - name: Download AxonFramework artifacts + working-directory: temp + run: | + mkdir -p AxonFramework-${{ env.AXON_FRAMEWORK_VERSION }} + cd AxonFramework-${{ env.AXON_FRAMEWORK_VERSION }} + echo "Working directory: $( pwd -P )" + ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }} + - name: Analyze AxonFramework working-directory: temp # Shell type can be skipped if jupyter notebook reports (and therefore conda) aren't needed shell: bash -el {0} - run: ./../scripts/analysis/analyze.sh --name AxonFramework --version ${{ env.AXON_FRAMEWORK_VERSION }} --report All --profile Neo4jv5 env: NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }} + run: | + ./../scripts/analysis/analyze.sh --report All --profile Neo4jv5 - name: Move reports from the temp to the results directory preserving their surrounding directory working-directory: temp diff --git a/COMMANDS.md b/COMMANDS.md index 457e6bdb3..34151d41d 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -2,28 +2,62 @@ ## Start an analysis -Use the following commands in the root directory of this repository to start an analysis manually e.g. for [AxonFramework](./scripts/artifacts/downloadAxonFramework.sh). +1. Create a directory for all analysis projects -```shell -export NEO4J_INITIAL_PASSWORD=theinitialpasswordthatihavechosenforneo4j -mkdir -p ./temp -cd temp -./../scripts/analysis/analyze.sh --name AxonFramework --version 4.8.0 -``` + ```shell + mkdir temp + cd temp + ``` + +1. Create a working directory for your specific analysis + + ```shell + mkdir MyFirstAnalysis + cd MyFirstAnalysis + ``` + +1. Choose an initial password for Neo4j + + ```shell + export NEO4J_INITIAL_PASSWORD=theinitialpasswordthatihavechosenforneo4j + ``` + +1. Create the `artifacts` directory for the code to be analyzed (without `cd` afterwards) + + ```shell + mkdir artifacts + ``` -Add the command line argument `--report Csv` to only run the CSV reports when you don't have Python set up -or want to skip Jupyter Notebooks. +1. Move the artifacts you want to analyze into the `artifacts` directory -Add the command line argument `--profile Neo4jv4` if you want to use the older long term support (june 2023) -version v4.4.x of Neo4j and compatible versions of plugins and JQAssistant. +1. Optionally run a predefined script to download artifacts + + ```shell + ./../../scripts/downloader/downloadAxonFramework.sh + ``` + +1. Optionally use a script to download artifacts from Maven ([details](#download-maven-artifacts-to-analyze)) + +1. Start the analysis + + ```shell + ./../../scripts/analysis/analyze.sh + ``` + +📖 +See [scripts/examples/analyzeAxonFramework.sh](./scripts/examples/analyzeAxonFramework.sh) as an example for all the above steps. See [code-reports Pipeline](./.github/workflows/code-reports.yml) on how to do this within a GitHub Actions Workflow. + +ⓘ +Add the command line argument `--report Csv` of [analyze.sh](./scripts/analysis/analyze.sh) to only generate CSV reports. This speeds up the report generation and doesn't depend on Python, Jupyter Notebook or any other related dependencies. + +ⓘ +Add the command line argument `--profile Neo4jv4` of [analyze.sh](./scripts/analysis/analyze.sh) if you want to use the older long term support (june 2023) version v4.4.x of Neo4j and compatible versions of plugins and JQAssistant. ### Notes -- Be sure to use Java 11 (Mai 2023 Neo4j v4 requirement) or Java 17 (June 2023 Neo4j v5 and jQAssistant CLI v2) +- Be sure to use Java 17 for Neo4j v5 and Java 11 for Neo4j v4 - Use your own initial Neo4j password - For more details have a look at the script [analyze.sh](./scripts/analysis/analyze.sh) -- The script file names (without the prefix "download" and without the file extension) in the directory [scripts/artifacts](./scripts/artifacts) provide all analysis names that are available. - Have a look at [code-reports.yml](./.github/workflows/code-reports.yml) for all details about setup steps and full automation. diff --git a/README.md b/README.md index 7e57640a8..3b4f9b6b6 100644 --- a/README.md +++ b/README.md @@ -58,10 +58,10 @@ The [Code Reports Pipeline](./.github/workflows/code-reports.yml) utilizes [GitH - [Checkout GIT Repository](https://github.com/actions/checkout) - [Setup Java](https://github.com/actions/setup-java) - [Setup Python with Conda](https://github.com/conda-incubator/setup-miniconda) package manager [Mambaforge](https://github.com/conda-forge/miniforge#mambaforge) +- Download artifacts that contain the code to be analyzed [scripts/artifacts](./scripts/downloader/) - Setup [Neo4j](https://neo4j.com) Graph Database ([analysis.sh](./scripts/analysis/analyze.sh)) - Setup [jQAssistant](https://jqassistant.org/get-started) for Java Analysis ([analysis.sh](./scripts/analysis/analyze.sh)) - Start [Neo4j](https://neo4j.com) Graph Database ([analysis.sh](./scripts/analysis/analyze.sh)) -- Trigger Artifacts download that contain the code to be analyzed [scripts/artifacts](./scripts/artifacts/) - Generate CSV Reports [scripts/reports](./scripts/reports) using the command line JSON parser [jq](https://jqlang.github.io/jq) - Generate [Jupyter Notebook](https://jupyter.org) reports using these libraries specified in the [environment.yml](./jupyter/environment.yml): - [Python](https://www.python.org) @@ -97,7 +97,7 @@ The [Code Reports Pipeline](./.github/workflows/code-reports.yml) utilizes [GitH ## 🤔 Questions & Answers - How can i run an analysis locally? - 👉 See [start-an-analysis](./COMMANDS.md#start-an-analysis) in the [Commands Reference](./COMMANDS.md). + 👉 See [Start an analysis](./COMMANDS.md#start-an-analysis) in the [Commands Reference](./COMMANDS.md). - How can i add an CSV report to the pipeline? 👉 Put your new cypher query into the [cypher](./cypher) directory or a suitable (new) sub directory. @@ -109,12 +109,12 @@ The [Code Reports Pipeline](./.github/workflows/code-reports.yml) utilizes [GitH 👉 Create a new Jupyter report script in the [scripts/reports](./scripts/reports/) directory. Take [OverviewJupyter.sh](./scripts/reports/OverviewJupyter.sh) as a reference for example. 👉 The script will automatically be included because of the directory and its name ending with "Jupyter.sh". -- How can i add another code base to analyze? - 👉 Create an new artifacts download script in the [scripts/artifacts](./scripts/artifacts) directory. Take [downloadAxonFramework.](./scripts/artifacts/downloadAxonFramework.sh) as a reference for example. - 👉 The script will be triggered when the [analyze](./scripts/analysis/analyze.sh) command +- How can i add another code basis to be analyzed automatically? + 👉 Create a new artifacts download script in the [scripts/artifacts](./scripts/artifacts) directory. Take for example [downloadAxonFramework.sh](./scripts/downloader/downloadAxonFramework.sh) as a reference. + 👉 Run the script separately before executing [analyze.sh](./scripts/analysis/analyze.sh) also in the [pipeline](./.github/workflows/code-reports.yml). - How can i trigger a full rescan of all artifacts? - 👉 Delete the file `artifactsChangeDetectionHash.txt` in the temporary `artifacts` directory. + 👉 Delete the file `artifactsChangeDetectionHash.txt` in the `artifacts` directory. - How can PDF generation be skipped to speed up report generation and not depend on chromium? 👉 Set environment variable `SKIP_JUPYTER_NOTEBOOK_PDF_GENERATION` to anything except an empty string. Example: diff --git a/scripts/SCRIPTS.md b/scripts/SCRIPTS.md index 639c68f17..a699ec1af 100644 --- a/scripts/SCRIPTS.md +++ b/scripts/SCRIPTS.md @@ -6,10 +6,10 @@ It provides a table listing each script file and its corresponding description f Script | Directory | Description -------|-----------|------------ | [analyze.sh](./analysis/analyze.sh) | analysis | Coordinates the end-to-end analysis process, encompassing tool installation, graph generation, and report generation. | -| [downloadAxonFramework.sh](./artifacts/downloadAxonFramework.sh) | artifacts | Downloads AxonFramework (https://developer.axoniq.io/axon-framework) artifacts from Maven Central. | | [copyReportsIntoResults.sh](./copyReportsIntoResults.sh) | | Copies the results from the temp directory to the results directory grouped by the analysis name. | | [detectChangedArtifacts.sh](./detectChangedArtifacts.sh) | | Detect changed files in the artifacts directory with a text file containing the last hash code of the contents. | | [downloadMavenArtifact.sh](./downloadMavenArtifact.sh) | | Downloads an artifact from Maven Central (https://mvnrepository.com/repos/central) | +| [downloadAxonFramework.sh](./downloader/downloadAxonFramework.sh) | downloader | Downloads AxonFramework (https://developer.axoniq.io/axon-framework) artifacts from Maven Central. | | [executeJupyterNotebook.sh](./executeJupyterNotebook.sh) | | Executes all steps in the given Jupyter Notebook (ipynb), stores it and converts it to Markdown (md) and PDF. | | [executeQuery.sh](./executeQuery.sh) | | Utilizes Neo4j's HTTP API to execute a Cypher query from an input file and provides the results in CSV format. | | [executeQueryFunctions.sh](./executeQueryFunctions.sh) | | Provides functions to execute Cypher queries using either "executeQuery.sh" or Neo4j's "cypher-shell". | diff --git a/scripts/analysis/analyze.sh b/scripts/analysis/analyze.sh index c7366c929..64b4cfd4b 100755 --- a/scripts/analysis/analyze.sh +++ b/scripts/analysis/analyze.sh @@ -2,21 +2,14 @@ # Coordinates the end-to-end analysis process, encompassing tool installation, graph generation, and report generation. # - Download and setup Neo4j and JQAssistant -# - Trigger artifacts download -# - Scan and analyze the artifacts to create the graph +# - Scan and analyze the contents of the artifacts directory to create the graph # - Trigger all requested reports -# Note: Everything is done in the current directory. -# It is recommended to create an empty directory (preferrable "temp") +# Note: Everything is done in the current (=working) directory and one directory above (shared downloads). +# It is recommended to create an empty directory (preferrable "temp") and +# within that another one for the analysis (e.g. "MyCodebaseName-Version") # and change into it prior to starting this script. -# Note: The argument "--name" is requried. It is used to create the working directory -# as well as to find the script "scripts/artifacts/download.sh" to download the artifacts. - -# Note: The argument "--version" is also required. -# It is appended to the working directory to further distinguish the results. -# The version is passed to the artifacts download script as an argument. - # Note: The argument "--report" is optional. The default value is "All". # It selects the report compilation, a named group of reports. Besides the default "All" there are e.g. "Csv" and "Jupyter". # This makes it possible to run only a subset of the reports. For example "Csv" won't need python to be set up and runs therefore much faster. @@ -29,17 +22,17 @@ # Note: The script and its sub scripts are designed to be as efficient as possible # when it comes to subsequent executions. -# Existing downloads, installations, artifacts, scans and processes will be detected. +# Existing downloads, installations, scans and processes will be detected. # Overrideable variables with directory names -ARTIFACT_SCRIPTS_DIRECTORY=${ARTIFACT_SCRIPTS_DIRECTORY:-"artifacts"} REPORTS_SCRIPTS_DIRECTORY=${REPORTS_SCRIPTS_DIRECTORY:-"reports"} REPORT_COMPILATIONS_SCRIPTS_DIRECTORY=${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY:-"compilations"} SETTINGS_PROFILE_SCRIPTS_DIRECTORY=${SETTINGS_PROFILE_SCRIPTS_DIRECTORY:-"profiles"} +ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"} # Function to display script usage usage() { - echo "Usage: $0 --name --version [--report ] [--profile ]" + echo "Usage: $0 [--report ] [--profile ]" exit 1 } @@ -51,14 +44,6 @@ settingsProfile="Default" while [[ $# -gt 0 ]]; do key="$1" case $key in - --name) - analysisName="${2}" - shift - ;; - --version) - analysisVersion="$2" - shift - ;; --report) analysisReportCompilation="$2" shift @@ -75,101 +60,69 @@ while [[ $# -gt 0 ]]; do shift done -# Check if the name is provided -if [[ -z ${analysisName} ]]; then - echo "analyze ${analysisName}: Error: Name is required." - usage -fi - -# Check if the version is provided -if [[ -z ${analysisVersion} ]]; then - echo "analyze ${analysisName}: Error: Version is required." - usage -fi - -# Assure that the analysis name only consists of letters and numbers -if ! [[ ${analysisName} =~ ^[[:alnum:]]+$ ]]; then - echo "analyze ${analysisName}: Error: Name can only contain letters and numbers." - exit 1 -fi - # Assure that the analysis report compilation only consists of letters and numbers if ! [[ ${analysisReportCompilation} =~ ^[[:alnum:]]+$ ]]; then - echo "analyze ${analysisName}: Report can only contain letters and numbers." + echo "analyze: Report can only contain letters and numbers." exit 1 fi # Assure that the settings profile only consists of letters and numbers if ! [[ ${settingsProfile} =~ ^[[:alnum:]]+$ ]]; then - echo "analyze ${analysisName}: Error: Settings profile can only contain letters and numbers." + echo "analyze: Error: Settings profile can only contain letters and numbers." exit 1 fi +# Check if Neo4j is installed +if [ ! -d "${ARTIFACTS_DIRECTORY}" ] ; then + echo "analyze: The ${ARTIFACTS_DIRECTORY} directory doesn't exist. Please download artifacts first." + exit 1 +fi + ## Get this "scripts/analysis" directory if not already set # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. # This way non-standard tools like readlink aren't needed. ANALYSIS_SCRIPT_DIR=${ANALYSIS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} -echo "analyze ${analysisName}: ANALYSIS_SCRIPT_DIR=${ANALYSIS_SCRIPT_DIR}" +echo "analyze: ANALYSIS_SCRIPT_DIR=${ANALYSIS_SCRIPT_DIR}" # Get the "scripts" directory by taking the path of this script and going one directory up. SCRIPTS_DIR=${SCRIPTS_DIR:-$(dirname -- "${ANALYSIS_SCRIPT_DIR}")} -echo "analyze ${analysisName}: SCRIPTS_DIR=${SCRIPTS_DIR}" - -# Assure that there is a download script for the given analysis name argument. -DOWNLOAD_SCRIPT="${SCRIPTS_DIR}/${ARTIFACT_SCRIPTS_DIRECTORY}/download${analysisName}.sh" -if [ ! -f "${DOWNLOAD_SCRIPT}" ] ; then - echo "analyze ${analysisName}: Error: No download${analysisName}.sh script in the directory ${SCRIPTS_DIR}/${ARTIFACT_SCRIPTS_DIRECTORY} for analysis name ${analysisName}." - exit 1 -fi +echo "analyze: SCRIPTS_DIR=${SCRIPTS_DIR}" # Assure that there is a report compilation script for the given report argument. REPORT_COMPILATION_SCRIPT="${SCRIPTS_DIR}/${REPORTS_SCRIPTS_DIRECTORY}/${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY}/${analysisReportCompilation}Reports.sh" if [ ! -f "${REPORT_COMPILATION_SCRIPT}" ] ; then - echo "analyze ${analysisName}: Error: No ${analysisReportCompilation}Reports.sh script in the directory ${SCRIPTS_DIR}/${REPORTS_SCRIPTS_DIRECTORY}/${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY} for report name ${analysisReportCompilation}." + echo "analyze: Error: No ${analysisReportCompilation}Reports.sh script in the directory ${SCRIPTS_DIR}/${REPORTS_SCRIPTS_DIRECTORY}/${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY} for report name ${analysisReportCompilation}." exit 1 fi # Assure that there is a script file for the given settings profile name. SETTINGS_PROFILE_SCRIPT="${SCRIPTS_DIR}/${SETTINGS_PROFILE_SCRIPTS_DIRECTORY}/${settingsProfile}.sh" if [ ! -f "${SETTINGS_PROFILE_SCRIPT}" ] ; then - echo "analyze ${analysisName}: Error: No ${settingsProfile}.sh script in the directory ${SCRIPTS_DIR}/${SETTINGS_PROFILE_SCRIPTS_DIRECTORY} for settings profile ${settingsProfile}." + echo "analyze: Error: No ${settingsProfile}.sh script in the directory ${SCRIPTS_DIR}/${SETTINGS_PROFILE_SCRIPTS_DIRECTORY} for settings profile ${settingsProfile}." exit 1 fi # Execute the settings profile script that sets all the neccessary settings variables (overrideable by environment variables). -echo "analyze ${analysisName}: Using analysis settings profile script ${SETTINGS_PROFILE_SCRIPT}" -source "${SETTINGS_PROFILE_SCRIPT}" || exit 1 - -# Create working directory if it hadn't been created yet -workingDirectory="${analysisName}-${analysisVersion}" -mkdir -p "${workingDirectory}" || exit 2 # Create the working directory only if it doesn't exist -pushd "${workingDirectory}" || exit 2 # Change into the working directory and remember the previous directory -echo "analyze ${analysisName}: Working Directory: $( pwd -P )" +echo "analyze: Using analysis settings profile script ${SETTINGS_PROFILE_SCRIPT}" +source "${SETTINGS_PROFILE_SCRIPT}" || exit 2 # Setup Tools source "${SCRIPTS_DIR}/setupNeo4j.sh" || exit 3 source "${SCRIPTS_DIR}/setupJQAssistant.sh" || exit 3 source "${SCRIPTS_DIR}/startNeo4j.sh" || exit 3 -# Execute the script "scripts/artifacts/download" >&2 + exit 1 +fi +artifactsVersion=$1 + +# Check if environment variable is set +if [ -z "${NEO4J_INITIAL_PASSWORD}" ]; then + echo "analyzeAxonFramework: Error: Requires environment variable NEO4J_INITIAL_PASSWORD to be set first. Use 'export NEO4J_INITIAL_PASSWORD='." + exit 1 +fi + +# Create the temporary directory for all analysis projects. +mkdir -p ./temp || exit 1 +cd ./temp || exit 1 + +# Create the working directory for this specific analysis. +mkdir -p "./AxonFramework-${artifactsVersion}" || exit 2 +cd "./AxonFramework-${artifactsVersion}" || exit 2 + +# Create the artifacts directory that will contain the code to be analyzed. +mkdir -p ./artifacts + +# Download AxonFramework artifacts (jar files) from Maven +./../../scripts/downloader/downloadAxonFramework.sh "${artifactsVersion}" + +# Start the analysis +./../../scripts/analysis/analyze.sh \ No newline at end of file From 3dd46ae52ed07d60c0c2b2b43aa14e236bcce870 Mon Sep 17 00:00:00 2001 From: JohT Date: Sat, 1 Jul 2023 17:04:49 +0200 Subject: [PATCH 3/4] Print change detection return value for debugging --- .github/workflows/code-reports.yml | 4 ++-- scripts/resetAndScanChanged.sh | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code-reports.yml b/.github/workflows/code-reports.yml index 3c8c6990d..b408ed5d6 100644 --- a/.github/workflows/code-reports.yml +++ b/.github/workflows/code-reports.yml @@ -106,13 +106,13 @@ jobs: ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }} - name: Analyze AxonFramework - working-directory: temp + working-directory: temp/AxonFramework-${{ env.AXON_FRAMEWORK_VERSION }} # Shell type can be skipped if jupyter notebook reports (and therefore conda) aren't needed shell: bash -el {0} env: NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }} run: | - ./../scripts/analysis/analyze.sh --report All --profile Neo4jv5 + ./../../scripts/analysis/analyze.sh --report All --profile Neo4jv5 - name: Move reports from the temp to the results directory preserving their surrounding directory working-directory: temp diff --git a/scripts/resetAndScanChanged.sh b/scripts/resetAndScanChanged.sh index c436b2e0a..33056be73 100755 --- a/scripts/resetAndScanChanged.sh +++ b/scripts/resetAndScanChanged.sh @@ -12,9 +12,10 @@ SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" echo "resetAndScanChanged SCRIPTS_DIR=$SCRIPTS_DIR" # Scan and analyze Artifacts when they were changed -if [[ $( source $SCRIPTS_DIR/detectChangedArtifacts.sh ) == "0" ]] ; then - echo "Artifacts unchanged. Scan skipped." +changeDetectionReturnCode=$( source "$SCRIPTS_DIR/detectChangedArtifacts.sh" ) +if [[ "${changeDetectionReturnCode}" == "0" ]] ; then + echo "resetAndScanChanged: Artifacts unchanged. Scan skipped." else - echo "Resetting database and scanning artifacts" + echo "resetAndScanChanged: Detected change (${changeDetectionReturnCode}). Resetting database and scanning artifacts." source "${SCRIPTS_DIR}/resetAndScan.sh" fi \ No newline at end of file From 53d60458589d5bfae07958fad57f0533f9eb9cb1 Mon Sep 17 00:00:00 2001 From: JohT Date: Sat, 1 Jul 2023 20:20:01 +0200 Subject: [PATCH 4/4] Introduce explore mode to start a manual analysis --- COMMANDS.md | 19 +++++++++++-------- scripts/analysis/analyze.sh | 18 ++++++++++++++++-- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/COMMANDS.md b/COMMANDS.md index 34151d41d..a2c40a380 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -44,14 +44,19 @@ ./../../scripts/analysis/analyze.sh ``` -📖 -See [scripts/examples/analyzeAxonFramework.sh](./scripts/examples/analyzeAxonFramework.sh) as an example for all the above steps. See [code-reports Pipeline](./.github/workflows/code-reports.yml) on how to do this within a GitHub Actions Workflow. +👉 See [scripts/examples/analyzeAxonFramework.sh](./scripts/examples/analyzeAxonFramework.sh) as an example script that combines all the above steps. +👉 See [code-reports Pipeline](./.github/workflows/code-reports.yml) on how to do this within a GitHub Actions Workflow. -ⓘ -Add the command line argument `--report Csv` of [analyze.sh](./scripts/analysis/analyze.sh) to only generate CSV reports. This speeds up the report generation and doesn't depend on Python, Jupyter Notebook or any other related dependencies. +### Command Line Options -ⓘ -Add the command line argument `--profile Neo4jv4` of [analyze.sh](./scripts/analysis/analyze.sh) if you want to use the older long term support (june 2023) version v4.4.x of Neo4j and compatible versions of plugins and JQAssistant. +The [analyze.sh](./scripts/analysis/analyze.sh) command comes with these command line options: + +- `--report Csv` only generates CSV reports. This speeds up the report generation and doesn't depend on Python, Jupyter Notebook or any other related dependencies. The default value os `All` to generate all reports. `Jupiter` will only generate Jupyter Notebook reports. + +- `--profile Neo4jv4` uses the older long term support (june 2023) version v4.4.x of Neo4j and suitable compatible versions of plugins and JQAssistant. `Neo4jv5` will explicitly select the newest (june 2023) version 5.x of Neo4j. Without setting +a profile, the newest versions will be used. Profiles are scripts that can be found in the directory [scripts/profiles](./scripts/profiles/). + +- `--explore` activates the "explore" mode where no reports are generated. Furthermore, Neo4j won't be stopped at the end of the script and will therefore continue running. This makes it easy to just set everything up but then use the running Neo4j server to explore the data manually. ### Notes @@ -59,8 +64,6 @@ Add the command line argument `--profile Neo4jv4` of [analyze.sh](./scripts/anal - Use your own initial Neo4j password - For more details have a look at the script [analyze.sh](./scripts/analysis/analyze.sh) -Have a look at [code-reports.yml](./.github/workflows/code-reports.yml) for all details about setup steps and full automation. - ## Generate Markdown References ### Update Cypher Reference diff --git a/scripts/analysis/analyze.sh b/scripts/analysis/analyze.sh index 64b4cfd4b..4f50c56d0 100755 --- a/scripts/analysis/analyze.sh +++ b/scripts/analysis/analyze.sh @@ -20,6 +20,10 @@ # This makes it possible to run an analysis with e.g. Neo4j v4 instead of v5. Further profiles might come in future. # Implemented is this as a script in "scripts/profiles" that starts with the settings profile name followed by ".sh". +# Note: The argument "--explore" is optional. It is a switch that is deactivated by default. +# It activates "explore" mode where no reports are executed and Neo4j keeps running (skip stop step). +# This makes it easy to just set everything up but then use the running Neo4j server to explore the data manually. + # Note: The script and its sub scripts are designed to be as efficient as possible # when it comes to subsequent executions. # Existing downloads, installations, scans and processes will be detected. @@ -32,13 +36,14 @@ ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"} # Function to display script usage usage() { - echo "Usage: $0 [--report ] [--profile ]" + echo "Usage: $0 [--report ] [--profile ] [--explore]" exit 1 } # Default values analysisReportCompilation="All" settingsProfile="Default" +exploreMode=false # Parse command line arguments while [[ $# -gt 0 ]]; do @@ -52,6 +57,10 @@ while [[ $# -gt 0 ]]; do settingsProfile="$2" shift ;; + --explore) + exploreMode=true + shift + ;; *) echo "analyze: Error: Unknown option: ${key}" usage @@ -118,10 +127,15 @@ source "${SCRIPTS_DIR}/resetAndScanChanged.sh" || exit 4 # Prepare and validate graph database before analyzing and creating reports source "${SCRIPTS_DIR}/prepareAnalysis.sh" || exit 5 +if ${exploreMode}; then + echo "analyze: Explore mode activated. Report generation will be skipped. Neo4j keeps running." + exit 0 +fi + ######################### # Create Reports ######################### -echo "Creating Reports with ${REPORT_COMPILATION_SCRIPT} ..." +echo "analyze: Creating Reports with ${REPORT_COMPILATION_SCRIPT} ..." source "${REPORT_COMPILATION_SCRIPT}" || exit 6 # Stop Neo4j at the end