Support Python's build-in module venv for virtual environments

Author: JohT

.github/workflows/internal-java-code-analysis-venv-draft.yml

@@ -0,0 +1,124 @@
+name: (DRAFT) Java Code Structure Graph Analysis using venv
+
+on:
+  push:
+    branches:
+      - main
+    # Ignore changes in documentation, general configuration and reports for push events
+    paths-ignore: 
+      - '**/*.md'
+      - '**/*.txt'
+      - '**/*.css'
+      - '**/*.html'
+      - '**/*.js'
+      - '.gitignore'
+      - '.gitattributes'
+      - 'renovate.json'
+      - 'renovate-presets/**'
+      - 'changelogTemplate.mustache'
+      - '**.code-workspace'
+      - '.github/workflows/internal-typescript-code-analysis.yml'
+      - '.github/workflows/*documentation.yml'
+  pull_request:
+    branches:
+      - main
+    # Ignore changes in documentation, general configuration and reports for pull request events
+    paths-ignore: 
+      - '**/*.md'
+      - '**/*.txt'
+      - '**/*.css'
+      - '**/*.html'
+      - '**/*.js'
+      - '.gitignore'
+      - '.gitattributes'
+      - 'renovate.json'
+      - 'renovate-presets/**'
+      - 'changelogTemplate.mustache'
+      - '**.code-workspace'
+      - '.github/workflows/internal-typescript-code-analysis.yml'
+      - '.github/workflows/*documentation.yml'
+
+# Requires the secret NEO4J_INITIAL_PASSWORD to be configured
+jobs:
+  prepare-code-to-analyze:
+    runs-on: ubuntu-latest
+    outputs:
+      analysis-name: ${{ steps.set-analysis-name.outputs.analysis-name }}
+      sources-upload-name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
+      artifacts-upload-name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }}
+
+    env: 
+      PROJECT_NAME: AxonFramework
+      # Version variable names matches renovate.json configuration entry
+      AXON_FRAMEWORK_VERSION: 4.11.2
+
+    steps:
+    - name: Checkout GIT Repository
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+    
+    - name: Run script tests
+      id: script-tests
+      run: ./scripts/runTests.sh
+
+    - name: Set Set output variable 'analysis-name'
+      id: set-analysis-name
+      run: echo "analysis-name=${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"
+
+    - name: Setup temp directory if missing
+      run: mkdir -p ./temp
+
+    - name: Download ${{ steps.set-analysis-name.outputs.analysis-name }}
+      working-directory: temp
+      run: |
+        mkdir -p ${{ steps.set-analysis-name.outputs.analysis-name }}
+        cd ${{ steps.set-analysis-name.outputs.analysis-name }}
+        echo "Working directory: $( pwd -P )"
+        ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }}
+
+    - name: Debug folder structure in temp directory
+      if: runner.debug == '1'
+      working-directory: temp
+      run: |
+        ls -R | grep ":$" | sed -e 's/:$//' -e 's/[^-][^\/]*\//--/g' -e 's/^/   /' -e 's/-/|/'
+ 
+    - name: (Prepare Code to Analyze) Generate ARTIFACT_UPLOAD_ID
+      run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV
+ 
+    - name: (Prepare Code to Analyze) Set sources-upload-name
+      id: set-sources-upload-name
+      run: echo "sources-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-sources_input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
+ 
+    - name: (Prepare Code to Analyze) Set output variable 'artifacts-upload-name'
+      id: set-artifacts-upload-name
+      run: echo "artifacts-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-artifacts-input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
+
+    - name: (Prepare Code to Analyze) Upload sources to analyze
+      if: success()
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      with:
+        name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
+        path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source
+        include-hidden-files: true
+        if-no-files-found: error
+        retention-days: 1
+
+    - name: (Prepare Code to Analyze) Upload artifacts to analyze
+      if: success()
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      with:
+        name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }}
+        path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/artifacts
+        if-no-files-found: error
+        retention-days: 1
+
+
+
+  analyze-code-graph:
+    needs: [prepare-code-to-analyze]
+    uses: ./.github/workflows/public-analyze-code-graph.yml
+    with:
+      analysis-name: ${{ needs.prepare-code-to-analyze.outputs.analysis-name }}
+      artifacts-upload-name: ${{ needs.prepare-code-to-analyze.outputs.artifacts-upload-name }}
+      sources-upload-name: ${{ needs.prepare-code-to-analyze.outputs.sources-upload-name }}
+      jupyter-pdf: "false"
+      use-venv_virtual_python_environment: "true"

.github/workflows/public-analyze-code-graph.yml

@@ -61,6 +61,12 @@ on:
         required: false
         type: string
         default: 'true'
+      use-venv_virtual_python_environment:
+        description: >
+          Use venv for virtual Python environments instead of Conda ("true") or not ("false", default).
+        required: false
+        type: string
+        default: 'false'
     outputs:
       uploaded-analysis-results:
         description: >
@@ -103,6 +109,7 @@ jobs:
 
       # "Setup Python" can be skipped if jupyter notebook analysis-results aren't needed
       - name: (Python Setup) Use version ${{ matrix.python }} with Conda package manager Miniforge
+        if: inputs.use-venv_virtual_python_environment == 'false'
         id: prepare-conda-environment
         uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3
         with:
@@ -112,7 +119,15 @@ jobs:
           environment-file: ./conda-environment.yml
           auto-activate-base: false
           show-channel-urls: true
+
+      - name: (Python Setup) Use version ${{ matrix.python }} with venv environment management module
+        if: inputs.use-venv_virtual_python_environment == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
       - name: (Python Setup) Conda environment info
+        if: inputs.use-venv_virtual_python_environment == 'false'
         shell: bash -el {0}
         run: | 
           conda info
@@ -168,6 +183,7 @@ jobs:
           ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: ${{ inputs.jupyter-pdf }}
           IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or ""
           PREPARE_CONDA_ENVIRONMENT: "false" # Had already been done in step with id "prepare-conda-environment".
+          USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV: ${{ inputs.use-venv_virtual_python_environment }}
         run: |
           TYPESCRIPT_SCAN_HEAP_MEMORY=${{ inputs.typescript-scan-heap-memory }} ./../../scripts/analysis/analyze.sh ${{ inputs.analysis-arguments }}
     

.gitignore

@@ -98,6 +98,8 @@ __pycache__/
 
 # Python environments
 .conda
+.venv/
+*.pyc
 
 # Optuna (and other) Database data
 *.db

GETTING_STARTED.md

@@ -84,13 +84,13 @@ Use these optional command line options as needed:
   ./../../scripts/analysis/analyze.sh --report Csv
   ```
 
-- Jupyter notebook reports when Python and Conda are installed (and Chromium Browser for PDF generation):
+- Jupyter notebook reports when Python and Conda (or venv) are installed (and Chromium Browser for PDF generation):
 
   ```shell
   ./../../scripts/analysis/analyze.sh --report Jupyter
   ```
 
-- Python reports when Python and Conda are installed (without Chromium Browser for PDF generation):
+- Python reports when Python and Conda (or venv) are installed (without Chromium Browser for PDF generation):
 
   ```shell
   ./../../scripts/analysis/analyze.sh --report Python
@@ -102,7 +102,7 @@ Use these optional command line options as needed:
   ./../../scripts/analysis/analyze.sh --report Visualization
   ```
 
-- All reports with Python, Conda, Node.js and npm installed:
+- All reports with Python, Conda (or venv), Node.js and npm installed:
 
   ```shell
   ./../../scripts/analysis/analyze.sh

README.md

@@ -86,8 +86,10 @@ Here are some fully automated graph visualizations utilizing [GraphViz](https://
 
 ### Additional Prerequisites for Python and Jupyter Notebooks
 
-- Python is required for Jupyter Notebook reports.
-- A conda package manager like [Miniconda](https://docs.conda.io/projects/miniconda/en/latest) or [Anaconda](https://www.anaconda.com/download)(Recommended for Windows) is required for Jupyter Notebook reports.
+- Python is required for Jupyter Notebook and Python reports.
+- Either [Conda](https://docs.conda.io) or Python's build-in module [venv](https://docs.python.org/3/library/venv.html) a required as environment manager.
+- For Conda, use for example [Miniconda](https://docs.conda.io/projects/miniconda/en/latest) or [Anaconda](https://www.anaconda.com/download)(Recommended for Windows).
+- To use venv, no additional installation is needed. For that the environment variable `USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV` needs to be set to `'true'`.
 - Chromium will automatically be downloaded if needed for Jupyter Notebook PDF reports generation.
 
 ### Additional Prerequisites for Graph Visualization
@@ -131,6 +133,7 @@ The [Code Structure Analysis Pipeline](./.github/workflows/internal-java-code-an
 - [Checkout GIT Repository](https://github.com/actions/checkout)
 - [Setup Java](https://github.com/actions/setup-java)
 - [Setup Python with Conda](https://github.com/conda-incubator/setup-miniconda) package manager [Mambaforge](https://github.com/conda-forge/miniforge#mambaforge)
+- [Setup Python with venv](https://docs.python.org/3/library/venv.html)
 - Download artifacts and optionally source code that contain the code to be analyzed [scripts/downloader](./scripts/downloader)
 - Setup [Neo4j](https://neo4j.com) Graph Database ([analysis.sh](./scripts/analysis/analyze.sh))
 - Setup [jQAssistant](https://jqassistant.github.io/jqassistant/current) for Java and [Typescript](https://github.com/jqassistant-plugin/jqassistant-typescript-plugin) analysis ([analysis.sh](./scripts/analysis/analyze.sh))

requirements.txt

@@ -0,0 +1,32 @@
+# --- Core Python version ---
+# NOTE: Python version must be >= 3.12 for compatibility
+# This should be enforced by the user/environment, not pip
+
+# --- Core tools ---
+jupyter==1.1.*
+matplotlib==3.10.*
+nbconvert[webpdf]==7.16.*
+numpy==1.26.*
+pandas==2.2.*
+pip==25.0.*
+setuptools==75.8.* # opentsne uses sklearn.base uses joblib uses distutils missing in Python >= 12 (TODO use native openTSNE?) 
+typing-extensions==4.12.* # Needed for opentsne and Python >= 3.12
+
+# --- Visualization ---
+wordcloud==1.9.*
+monotonic==1.*
+plotly[kaleido]==6.2.*
+seaborn==0.13 # To visualize clustering results
+
+# --- Machine Learning / Optimization ---
+scikit-learn==1.6.*
+optuna==4.3.*
+umap-learn==0.5.* # Dimensionality reduction to visualize node embeddings in 2D
+
+# --- Database connector ---
+neo4j==5.23.*
+
+# --- Native/scientific packages (may require compilation) ---
+# These are included but may cause install errors in pip/venv
+opentsne==1.0.*       # Dimensionality reduction to visualize node embeddings in 2D. Might get replaced by umap.
+shap==0.48.*          # For e.g. explaining anomaly detection results

scripts/activateCondaEnvironment.sh

@@ -1,17 +1,33 @@
 #!/usr/bin/env bash
 
-# Activates the Conda (Python package manager) environment "codegraph" with all packages needed to execute the Jupyter Notebooks.
+# Activates the Conda (Python package manager) environment "codegraph" with all packages needed to run the included Jupyter Notebooks and Python scripts.
 
 # Note: This script uses the conda environment defined in CODEGRAPH_CONDA_ENVIRONMENT (defaults to "codegraph").
-#       If the environment hadn't been created yet, it will use "conda-environment.yml" from the root directory
-#       in the same directory as the given jupyter notebook ipynb file
-#       to create the environment.
+#       If the environment hadn't been created yet, it will use "conda-environment.yml" from the root directory to create the environment.
 
 # Requires operatingSystemFunctions.sh
 
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+PREPARE_CONDA_ENVIRONMENT=${PREPARE_CONDA_ENVIRONMENT:-"true"} # Wether to prepare a Python environment with Conda if needed (default, "true") or use an already prepared Conda environment ("false")
+
+if [ "${PREPARE_CONDA_ENVIRONMENT}" = "false" ]; then
+    echo "activateCondaEnvironment: Skipping activation. An already activated environment and installed dependencies are expected (PREPARE_CONDA_ENVIRONMENT=false)."
+    # "return" needs to be used here instead of "exit".
+    # This script is included in another script by using "source". 
+    # "exit" would end the main script, "return" just ends this sub script.
+    return 0
+fi 
+
+if [ "${USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV}" = "true" ]; then
+    echo "activateCondaEnvironment: Skipping activation. venv will be used instead of conda (USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV=true)."
+    # "return" needs to be used here instead of "exit".
+    # This script is included in another script by using "source". 
+    # "exit" would end the main script, "return" just ends this sub script.
+    return 0
+fi 
+
 ## Get this "scripts" directory if not already set
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -37,16 +53,6 @@ echo "activateCondaEnvironment: CONDA_PREFIX=${CONDA_PREFIX}"
 echo "activateCondaEnvironment: Current conda environment=${CONDA_DEFAULT_ENV}"
 echo "activateCondaEnvironment: Target conda environment=${CODEGRAPH_CONDA_ENVIRONMENT}"
 
-PREPARE_CONDA_ENVIRONMENT=${PREPARE_CONDA_ENVIRONMENT:-"true"} # Wether to prepare a Python environment with Conda if needed (default, "true") or use an already prepared Conda environment ("false")
-
-if [ "${PREPARE_CONDA_ENVIRONMENT}" = "false" ]; then
-    echo "activateCondaEnvironment: Skipping activation. ${PREPARE_CONDA_ENVIRONMENT} is set to false."
-    # "return" needs to be used here instead of "exit".
-    # This script is included in another script by using "source". 
-    # "exit" would end the main script, "return" just ends this sub script.
-    return 0
-fi 
-
 # Include operation system function to for example detect Windows.
 source "${SCRIPTS_DIR}/operatingSystemFunctions.sh"