Merge pull request #15 from JohT/feature/open-graph-data-science

Open graph-data-science

Author: JohT

.github/workflows/code-reports.yml

@@ -112,7 +112,7 @@ jobs:
       env:
         NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }}
       run: |
-        ./../../scripts/analysis/analyze.sh --report All --profile Neo4jv5
+        ./../../scripts/analysis/analyze.sh
 
     - name: Move reports from the temp to the results directory preserving their surrounding directory
       working-directory: temp
@@ -128,13 +128,15 @@ jobs:
         retention-days: 5
 
     # Upload Database Export
-    - name: Archive exported database
-      uses: actions/upload-artifact@v3
-      with:
-        name: code-report-database-export-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
-        path: ./temp/**/import
-        if-no-files-found: error
-        retention-days: 5
+    # Only possible after an export with "./../../scripts/analysis/analyze.sh --report DatabaseCsvExport"
+    # Won't be done here because of performance and security concerns
+    #- name: Archive exported database
+    #  uses: actions/upload-artifact@v3
+    #  with:
+    #    name: code-report-database-export-${{ matrix.java }}-python-${{ matrix.python }}-mambaforge-${{ matrix.mambaforge }}
+    #    path: ./temp/**/import
+    #    if-no-files-found: error
+    #    retention-days: 5
 
     # Commit and push the native image agent results
     - name: Display environment variable "github.event_name"

COMMANDS.md

@@ -51,7 +51,7 @@
 
 The [analyze.sh](./scripts/analysis/analyze.sh) command comes with these command line options:
 
-- `--report Csv` only generates CSV reports. This speeds up the report generation and doesn't depend on Python, Jupyter Notebook or any other related dependencies. The default value os `All` to generate all reports. `Jupiter` will only generate Jupyter Notebook reports.
+- `--report Csv` only generates CSV reports. This speeds up the report generation and doesn't depend on Python, Jupyter Notebook or any other related dependencies. The default value os `All` to generate all reports. `Jupiter` will only generate Jupyter Notebook reports. `DatabaseCsvExport` exports the whole graph database as a CSV file (performance intense, check if there are security concerns first).
 
 - `--profile Neo4jv4` uses the older long term support (june 2023) version v4.4.x of Neo4j and suitable compatible versions of plugins and JQAssistant. `Neo4jv5` will explicitly select the newest (june 2023) version 5.x of Neo4j. Without setting
 a profile, the newest versions will be used. Profiles are scripts that can be found in the directory [scripts/profiles](./scripts/profiles/).
@@ -123,6 +123,7 @@ to download a Maven artifact into the artifacts directory:
 - `-a <maven artifact name>`
 - `-v <maven artifact version>`
 - `-t <maven artifact type (optional, defaults to jar)>`
+- `-d <target directory for the downloaded file (optional, defaults to "artifacts")>`
 
 ### Reset the database and scan the java artifacts
 

renovate.json

@@ -66,6 +66,18 @@
       "depNameTemplate": "neo4j/graph-data-science",
       "datasourceTemplate": "github-releases"
     },
+    {
+      "fileMatch": [
+        "^scripts\/profiles\/Neo4jv5\\.sh$",
+        "^scripts\/profiles\/Default\\.sh$",
+        "^scripts\/[^\/]*\\.sh$"
+      ],
+      "matchStrings": [
+        "NEO4J_OPEN_GDS_PLUGIN_VERSION:-\\\"?(?<currentValue>.*?)\\\""
+      ],
+      "depNameTemplate": "JohT/open-graph-data-science-packaging",
+      "datasourceTemplate": "github-releases"
+    },
     {
       "fileMatch": [
         "^scripts\/profiles\/Neo4jv5\\.sh$",

scripts/SCRIPTS.md

@@ -8,8 +8,10 @@ Script | Directory | Description
 | [analyze.sh](./analysis/analyze.sh) | analysis | Coordinates the end-to-end analysis process, encompassing tool installation, graph generation, and report generation. |
 | [copyReportsIntoResults.sh](./copyReportsIntoResults.sh) |  | Copies the results from the temp directory to the results directory grouped by the analysis name. |
 | [detectChangedArtifacts.sh](./detectChangedArtifacts.sh) |  | Detect changed files in the artifacts directory with a text file containing the last hash code of the contents. |
+| [download.sh](./download.sh) |  | Downloads a file into the directory of the environment variable SHARED_DOWNLOADS_DIRECTORY (or default "../downloads"). |
 | [downloadMavenArtifact.sh](./downloadMavenArtifact.sh) |  | Downloads an artifact from Maven Central (https://mvnrepository.com/repos/central) |
 | [downloadAxonFramework.sh](./downloader/downloadAxonFramework.sh) | downloader | Downloads AxonFramework (https://developer.axoniq.io/axon-framework) artifacts from Maven Central. |
+| [analyzeAxonFramework.sh](./examples/analyzeAxonFramework.sh) | examples | This is an example for an analysis of AxonFramework  |
 | [executeJupyterNotebook.sh](./executeJupyterNotebook.sh) |  | Executes all steps in the given Jupyter Notebook (ipynb), stores it and converts it to Markdown (md) and PDF. |
 | [executeQuery.sh](./executeQuery.sh) |  | Utilizes Neo4j's HTTP API to execute a Cypher query from an input file and provides the results in CSV format. |
 | [executeQueryFunctions.sh](./executeQueryFunctions.sh) |  | Provides functions to execute Cypher queries using either "executeQuery.sh" or Neo4j's "cypher-shell".  |
@@ -22,7 +24,6 @@ Script | Directory | Description
 | [Neo4jv5.sh](./profiles/Neo4jv5.sh) | profiles | Sets all settings variables for an analysis with Neo4j v5.x (newest version as of june 2023). |
 | [CentralityCsv.sh](./reports/CentralityCsv.sh) | reports | Looks for centrality using the Graph Data Science Library of Neo4j and creates CSV reports. |
 | [CommunityCsv.sh](./reports/CommunityCsv.sh) | reports | Detects communities using the Graph Data Science Library of Neo4j and creates CSV reports. |
-| [DatabaseCsvExport.sh](./reports/DatabaseCsvExport.sh) | reports | Exports the whole graph database as a CSV file using the APOC procedure "apoc.export.csv.all" |
 | [ExternalDependenciesCsv.sh](./reports/ExternalDependenciesCsv.sh) | reports | Executes "Package_Usage" Cypher queries to get the "external-dependencies-csv" CSV reports. |
 | [ExternalDependenciesJupyter.sh](./reports/ExternalDependenciesJupyter.sh) | reports | Creates the "overview" report (ipynb, md, pdf) based on the Jupyter Notebook "Overview.ipynb". |
 | [InternalDependenciesCsv.sh](./reports/InternalDependenciesCsv.sh) | reports | Executes "Package_Usage" Cypher queries to get the "internal-dependencies" CSV reports. |
@@ -37,6 +38,7 @@ Script | Directory | Description
 | [WordcloudJupyter.sh](./reports/WordcloudJupyter.sh) | reports | Creates the "overview" report (ipynb, md, pdf) based on the Jupyter Notebook "Overview.ipynb". |
 | [AllReports.sh](./reports/compilations/AllReports.sh) | compilations | Runs all report scripts. |
 | [CsvReports.sh](./reports/compilations/CsvReports.sh) | compilations | Runs all CSV report scripts (no Python and Chromium required). |
+| [DatabaseCsvExportReports.sh](./reports/compilations/DatabaseCsvExportReports.sh) | compilations | Exports the whole graph database as a CSV file using the APOC procedure "apoc.export.csv.all" |
 | [JupyterReports.sh](./reports/compilations/JupyterReports.sh) | compilations | Runs all Jupyter Notebook report scripts. |
 | [resetAndScan.sh](./resetAndScan.sh) |  | Deletes all data in the Neo4j graph database and rescans the downloaded artifacts to create a new graph. |
 | [resetAndScanChanged.sh](./resetAndScanChanged.sh) |  | Executes "resetAndScan.sh" only if "detectChangedArtifacts.sh" returns detected changes. |

scripts/analysis/analyze.sh

@@ -28,6 +28,8 @@
 #       when it comes to subsequent executions.
 #       Existing downloads, installations, scans and processes will be detected.
 
+# Requires setupNeo4j.sh,setupJQAssistant.sh,startNeo4j.sh,resetAndScanChanged.sh,prepareAnalysis.sh,stopNeo4j.sh,comilations/*.sh,profiles/*.sh
+
 # Overrideable variables with directory names
 REPORTS_SCRIPTS_DIRECTORY=${REPORTS_SCRIPTS_DIRECTORY:-"reports"}
 REPORT_COMPILATIONS_SCRIPTS_DIRECTORY=${REPORT_COMPILATIONS_SCRIPTS_DIRECTORY:-"compilations"}

scripts/copyReportsIntoResults.sh

@@ -6,6 +6,8 @@
 
 # Notice that this scripts needs to be executed within the "temp" directory.
 
+# Requires generateMarkdownReference.sh
+
 ## Get this "scripts" directory if not already set
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.

scripts/download.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+# Downloads a file into the directory of the environment variable SHARED_DOWNLOADS_DIRECTORY (or default "../downloads").
+# Does nothing if the file already exists.
+
+# Command line options:
+# --url Download URL (required)
+# --filename Target file name with extension without path (optional, default = basename of download URL)
+
+# Function to display script usage
+usage() {
+  echo "Usage: $0 --url https://my.download.url [--filename download-file-name-without-path.ext> (default=url filename)]"
+  exit 1
+}
+
+# Default values
+downloadUrl=""
+filename=""
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+  key="$1"
+  case $key in
+    --url)
+      downloadUrl="$2"
+      shift
+      ;;
+    --filename)
+      filename="$2"
+      shift
+      ;;
+    *)
+      echo "download: Error: Unknown option: ${key}"
+      usage
+      ;;
+  esac
+  shift
+done
+
+if [[ -z ${downloadUrl} ]]; then
+  echo "${USAGE}"
+  exit 1
+fi
+
+if ! curl --head --fail ${downloadUrl} >/dev/null 2>&1; then
+  echo "download: Error: Invalid URL: ${downloadUrl}"
+  exit 1
+fi
+
+if [[ -z ${filename} ]]; then
+  filename=$(basename -- "${downloadUrl}")
+fi
+
+# Get shared download directory and create it if it doesn't exist
+SHARED_DOWNLOADS_DIRECTORY="${SHARED_DOWNLOADS_DIRECTORY:-$(dirname "$( pwd )")/downloads}"
+if [ ! -d "${SHARED_DOWNLOADS_DIRECTORY}" ] ; then
+  echo "download: Creating shared downloads directory ${SHARED_DOWNLOADS_DIRECTORY}"
+  mkdir -p ${SHARED_DOWNLOADS_DIRECTORY}
+fi
+
+# Download the file if it doesn't exist in the shared downloads directory 
+if [ ! -f "${SHARED_DOWNLOADS_DIRECTORY}/${filename}" ] ; then
+    echo "download: Downloading ${filename} from ${downloadUrl} into ${SHARED_DOWNLOADS_DIRECTORY}"
+
+    # Download the file
+    if ! curl -L --fail-with-body -o "${SHARED_DOWNLOADS_DIRECTORY}/${filename}" "${downloadUrl}"; then
+        echo "download: Error: Failed to download ${filename}"
+        rm -f "${SHARED_DOWNLOADS_DIRECTORY}/${filename}"
+        exit 1
+    fi
+else
+    echo "download: ${filename} already downloaded"
+fi
+
+# Check downloaded file size to be at least 600 bytes or otherwise delete the invalid file
+downloaded_file_size=$(wc -c "${SHARED_DOWNLOADS_DIRECTORY}/${filename}" | awk '{print $1}')
+if [[ "${downloaded_file_size}" -le 600 ]]; then
+    echo "download: Error: Failed to download ${filename}: Filesize: ${downloaded_file_size} < 600 bytes"
+    rm -f "${SHARED_DOWNLOADS_DIRECTORY}/${filename}"
+    exit 1
+fi
+
+# Fail if download failed
+if [ ! -f "${SHARED_DOWNLOADS_DIRECTORY}/${filename}" ] ; then
+    echo "download: Error: Failed to download ${filename}"
+    exit 1
+fi

scripts/downloadMavenArtifact.sh

@@ -6,66 +6,83 @@
 # -a Maven Artifact Name
 # -v Maven Artifact Version
 # -t Maven Artifact Type (defaults to jar)
+# -d Target directory for the downloaded file
 
-# Read options
-ARTIFACT_TYPE="jar"
+# Requires download.sh
+
+# Overrideable constants
+ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"}
+SHARED_DOWNLOADS_DIRECTORY="${SHARED_DOWNLOADS_DIRECTORY:-$(dirname "$( pwd )")/downloads}"
+
+# Default and initial values for command line options
+groupId=""
+artifactId=""
+version=""
+artifactType="jar"
+targetDirectory="${ARTIFACTS_DIRECTORY}"
+
+# Read  command line options
+USAGE="downloadMavenArtifact: Usage: $0 [-g group_id] [-a artifact_id] [-v version] [-t type (default=jar)] [-d targetDirectory (default=${ARTIFACTS_DIRECTORY})]"
 OPTIND=1
-while getopts "g:a:v:t:" opt; do
+while getopts "g:a:v:t:d:" opt; do
   case ${opt} in
     g )
-      GROUP_ID=${OPTARG}
+      groupId=${OPTARG}
       ;;
     a )
-      ARTIFACT_ID=${OPTARG}
+      artifactId=${OPTARG}
       ;;
     v )
-      VERSION=${OPTARG}
+      version=${OPTARG}
       ;;
     t )
-      ARTIFACT_TYPE=${OPTARG}
+      artifactType=${OPTARG}
+      ;;
+    d )
+      targetDirectory=${OPTARG}
       ;;
     \? )
-      echo "Usage: $0 [-g group_id] [-a artifact_id] [-v version] [-t type (default=jar)]"
+      echo "${USAGE}"
       exit 1
       ;;
   esac
 done
 
-if [[ -z ${GROUP_ID} || -z ${ARTIFACT_ID} || -z ${VERSION} || -z ${ARTIFACT_TYPE} ]]; then
-  echo "Usage: $0 [-g group_id] [-a artifact_id] [-v version] [-t type (default=jar)]"
+if [[ -z ${groupId} || -z ${artifactId} || -z ${version} || -z ${artifactType} || -z ${targetDirectory} ]]; then
+  echo "${USAGE}"
   exit 1
 fi
 
-# Overrideable constants
-ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"}
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )}
 
 # Internal constants
 BASE_URL="https://repo1.maven.org/maven2"
-ARTIFACT_FILENAME="${ARTIFACT_ID}-${VERSION}.${ARTIFACT_TYPE}"
-GROUP_ID_FOR_API="$(echo "${GROUP_ID}" | tr '.' '/')"
-DOWNLOAD_URL="${BASE_URL}/${GROUP_ID_FOR_API}/${ARTIFACT_ID}/${VERSION}/${ARTIFACT_FILENAME}"
-
-# Download Maven Artifact into the ARTIFACTS_DIRECTORY
-if [ ! -f "${ARTIFACTS_DIRECTORY}/${ARTIFACT_FILENAME}" ] ; then
-    echo "Downloading ${DOWNLOAD_URL}"
+ARTIFACT_FILENAME="${artifactId}-${version}.${artifactType}"
+GROUP_ID_FOR_API="$(echo "${groupId}" | tr '.' '/')"
+DOWNLOAD_URL="${BASE_URL}/${GROUP_ID_FOR_API}/${artifactId}/${version}/${ARTIFACT_FILENAME}"
 
-    # Download Maven Artifact
-    curl -L --fail-with-body -O "${DOWNLOAD_URL}"
+# Download Maven Artifact into the "targetDirectory"
+if [ ! -f "./${targetDirectory}/${ARTIFACT_FILENAME}" ] ; then
+    source ${SCRIPTS_DIR}/download.sh --url "${DOWNLOAD_URL}" || exit 1
 
-    # Create artifacts directory if it doen't exist
-    mkdir -p "${ARTIFACTS_DIRECTORY}"
+    # Create artifacts targetDirectory if it doen't exist
+    mkdir -p "./${targetDirectory}" || exit 1
 
     # Delete already existing older versions of the artifact
-    rm -f "${ARTIFACTS_DIRECTORY}/${ARTIFACT_ID}"*
+    rm -f "./${targetDirectory}/${artifactId}"* || exit 1
 
-    # Move artifact to artifacts directory
-    mv "${ARTIFACT_FILENAME}" "${ARTIFACTS_DIRECTORY}"
+    # Copy artifact into artifacts targetDirectory
+    cp -R "${SHARED_DOWNLOADS_DIRECTORY}/${ARTIFACT_FILENAME}" "./${targetDirectory}" || exit 1
 else
-    echo "${ARTIFACT_FILENAME} already downloaded"
+    echo "downloadMavenArtifact: ${ARTIFACT_FILENAME} already downloaded into target directory ${targetDirectory}"
 fi
 
 # Fail if Maven Download failed
-if [ ! -f "${ARTIFACTS_DIRECTORY}/${ARTIFACT_FILENAME}" ] ; then
-    echo "Failed to download ${ARTIFACT_FILENAME}"
+if [ ! -f "${targetDirectory}/${ARTIFACT_FILENAME}" ] ; then
+    echo "downloadMavenArtifact: Error: Failed to download ${ARTIFACT_FILENAME}"
     exit 1
 fi

scripts/downloader/downloadAxonFramework.sh

@@ -8,6 +8,8 @@
 
 # Note: This script is meant to be started within the temporary analysis directory (e.g. "temp/AnalysisName/")
 
+# Requires downloadMavenArtifact.sh
+
 # Get the analysis name from the middle part of the current file name (without prefix "download" and without extension)
 SCRIPT_FILE_NAME="$(basename -- "${BASH_SOURCE[0]}")"
 SCRIPT_FILE_NAME_WITHOUT_EXTENSION="${SCRIPT_FILE_NAME%%.*}"
@@ -30,11 +32,11 @@ echo "download${ANALYSIS_NAME}: ARTIFACTS_VERSION=${ARTIFACTS_VERSION}"
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
 # This way non-standard tools like readlink aren't needed.
-ANALYSIS_SCRIPT_DIR=${ANALYSIS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )}
-echo "download${ANALYSIS_NAME}: ANALYSIS_SCRIPT_DIR=${ANALYSIS_SCRIPT_DIR}"
+DOWNLOADER_SCRIPTS_DIR=${DOWNLOADER_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )}
+echo "download${ANALYSIS_NAME}: DOWNLOADER_SCRIPTS_DIR=${DOWNLOADER_SCRIPTS_DIR}"
 
 # Get the "scripts" directory by taking the path of this script and going one directory up.
-SCRIPTS_DIR=${SCRIPTS_DIR:-$(dirname -- "${ANALYSIS_SCRIPT_DIR}")}
+SCRIPTS_DIR=${SCRIPTS_DIR:-$(dirname -- "${DOWNLOADER_SCRIPTS_DIR}")}
 echo "download${ANALYSIS_NAME}: SCRIPTS_DIR=${SCRIPTS_DIR}"
 
 ################################################################

scripts/executeQueryFunctions.sh

@@ -2,6 +2,8 @@
 
 # Provides functions to execute Cypher queries using either "executeQuery.sh" or Neo4j's "cypher-shell". 
 
+# Requires executeQuery.sh
+
 ## Get this "scripts" directory if not already set
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.