Support Python's build-in module venv for virtual environments

Author: JohT

.github/workflows/internal-check-python-venv-support.yml

@@ -0,0 +1,42 @@
+name: Check Python venv virtual environment
+
+on:
+  pull_request:
+    branches:
+      - main
+    # Only watch changes related to Python virtual environment venv
+    paths:
+      - 'requirements.txt'
+      - 'scripts/activatePythonEnvironment.sh'
+      - '.github/workflows/internal-check-python-venv-support.yml' # or when this file changed
+
+jobs:
+  check-python-venv-environment:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - os: ubuntu-22.04
+          python: 3.12
+
+    steps:
+    - name: Checkout GIT Repository
+      uses: actions/checkout@v4
+
+    - name: (Python Setup) Use version ${{ matrix.python }} with venv environment management module
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+        cache: 'pip'
+    
+    - name: Create, activate, install and check virtual environment
+      env: 
+        USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV: "true"
+      # 1.  Run the script under test to create, activate and install the virtual environment
+      # 2a. Run pip in dry-run mode without installing or resolving dependencies
+      # 2b. Suppress all pip output (stderr)
+      # 2c. Check if pip *would install* anything using grep
+      # 2d. If there are missing dependencies and the environment is incomplete, return 1 (indicates all requirements already satisfied)
+      run: |
+        ./scripts/activatePythonEnvironment.sh
+        pip install --dry-run --no-deps --requirement "./requirements.txt" 2>/dev/null | grep -q "Would install" || return 1

.github/workflows/public-analyze-code-graph.yml

@@ -61,6 +61,12 @@ on:
         required: false
         type: string
         default: 'true'
+      use-venv_virtual_python_environment:
+        description: >
+          Use venv for virtual Python environments instead of Conda ("true") or not ("false", default).
+        required: false
+        type: string
+        default: 'false'
     outputs:
       uploaded-analysis-results:
         description: >
@@ -103,6 +109,7 @@ jobs:
 
       # "Setup Python" can be skipped if jupyter notebook analysis-results aren't needed
       - name: (Python Setup) Use version ${{ matrix.python }} with Conda package manager Miniforge
+        if: inputs.use-venv_virtual_python_environment == 'false'
         id: prepare-conda-environment
         uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3
         with:
@@ -112,7 +119,16 @@ jobs:
           environment-file: ./conda-environment.yml
           auto-activate-base: false
           show-channel-urls: true
+
+      - name: (Python Setup) Use version ${{ matrix.python }} with venv environment management module
+        if: inputs.use-venv_virtual_python_environment == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: 'pip'
+
       - name: (Python Setup) Conda environment info
+        if: inputs.use-venv_virtual_python_environment == 'false'
         shell: bash -el {0}
         run: | 
           conda info
@@ -168,6 +184,7 @@ jobs:
           ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: ${{ inputs.jupyter-pdf }}
           IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or ""
           PREPARE_CONDA_ENVIRONMENT: "false" # Had already been done in step with id "prepare-conda-environment".
+          USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV: ${{ inputs.use-venv_virtual_python_environment }}
         run: |
           TYPESCRIPT_SCAN_HEAP_MEMORY=${{ inputs.typescript-scan-heap-memory }} ./../../scripts/analysis/analyze.sh ${{ inputs.analysis-arguments }}
     

.gitignore

@@ -98,6 +98,8 @@ __pycache__/
 
 # Python environments
 .conda
+.venv/
+*.pyc
 
 # Optuna (and other) Database data
 *.db

GETTING_STARTED.md

@@ -84,13 +84,13 @@ Use these optional command line options as needed:
   ./../../scripts/analysis/analyze.sh --report Csv
   ```
 
-- Jupyter notebook reports when Python and Conda are installed (and Chromium Browser for PDF generation):
+- Jupyter notebook reports when Python and Conda (or venv) are installed (and Chromium Browser for PDF generation):
 
   ```shell
   ./../../scripts/analysis/analyze.sh --report Jupyter
   ```
 
-- Python reports when Python and Conda are installed (without Chromium Browser for PDF generation):
+- Python reports when Python and Conda (or venv) are installed (without Chromium Browser for PDF generation):
 
   ```shell
   ./../../scripts/analysis/analyze.sh --report Python
@@ -102,7 +102,7 @@ Use these optional command line options as needed:
   ./../../scripts/analysis/analyze.sh --report Visualization
   ```
 
-- All reports with Python, Conda, Node.js and npm installed:
+- All reports with Python, Conda (or venv), Node.js and npm installed:
 
   ```shell
   ./../../scripts/analysis/analyze.sh

README.md

@@ -86,8 +86,10 @@ Here are some fully automated graph visualizations utilizing [GraphViz](https://
 
 ### Additional Prerequisites for Python and Jupyter Notebooks
 
-- Python is required for Jupyter Notebook reports.
-- A conda package manager like [Miniconda](https://docs.conda.io/projects/miniconda/en/latest) or [Anaconda](https://www.anaconda.com/download)(Recommended for Windows) is required for Jupyter Notebook reports.
+- Python is required for Jupyter Notebook and Python reports.
+- Either [Conda](https://docs.conda.io) or Python's build-in module [venv](https://docs.python.org/3/library/venv.html) a required as environment manager.
+- For Conda, use for example [Miniconda](https://docs.conda.io/projects/miniconda/en/latest) or [Anaconda](https://www.anaconda.com/download)(Recommended for Windows).
+- To use venv, no additional installation is needed. For that the environment variable `USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV` needs to be set to `'true'`.
 - Chromium will automatically be downloaded if needed for Jupyter Notebook PDF reports generation.
 
 ### Additional Prerequisites for Graph Visualization
@@ -131,6 +133,7 @@ The [Code Structure Analysis Pipeline](./.github/workflows/internal-java-code-an
 - [Checkout GIT Repository](https://github.com/actions/checkout)
 - [Setup Java](https://github.com/actions/setup-java)
 - [Setup Python with Conda](https://github.com/conda-incubator/setup-miniconda) package manager [Mambaforge](https://github.com/conda-forge/miniforge#mambaforge)
+- [Setup Python with venv](https://docs.python.org/3/library/venv.html)
 - Download artifacts and optionally source code that contain the code to be analyzed [scripts/downloader](./scripts/downloader)
 - Setup [Neo4j](https://neo4j.com) Graph Database ([analysis.sh](./scripts/analysis/analyze.sh))
 - Setup [jQAssistant](https://jqassistant.github.io/jqassistant/current) for Java and [Typescript](https://github.com/jqassistant-plugin/jqassistant-typescript-plugin) analysis ([analysis.sh](./scripts/analysis/analyze.sh))

requirements.txt

@@ -0,0 +1,32 @@
+# --- Core Python version ---
+# NOTE: Python version must be >= 3.12 for compatibility
+# This should be enforced by the user/environment, not pip
+
+# --- Core tools ---
+jupyter==1.1.*
+matplotlib==3.10.*
+nbconvert[webpdf]==7.16.*
+numpy==1.26.*
+pandas==2.2.*
+pip==25.0.*
+setuptools==75.8.* # opentsne uses sklearn.base uses joblib uses distutils missing in Python >= 12 (TODO use native openTSNE?) 
+typing-extensions==4.12.* # Needed for opentsne and Python >= 3.12
+
+# --- Visualization ---
+wordcloud==1.9.*
+monotonic==1.*
+plotly[kaleido]==6.2.*
+seaborn==0.13 # To visualize clustering results
+
+# --- Machine Learning / Optimization ---
+scikit-learn==1.6.*
+optuna==4.3.*
+umap-learn==0.5.* # Dimensionality reduction to visualize node embeddings in 2D
+
+# --- Database connector ---
+neo4j==5.23.*
+
+# --- Native/scientific packages (may require compilation) ---
+# These are included but may cause install errors in pip/venv
+opentsne==1.0.*       # Dimensionality reduction to visualize node embeddings in 2D. Might get replaced by umap.
+shap==0.48.*          # For e.g. explaining anomaly detection results

scripts/activateCondaEnvironment.sh

@@ -1,17 +1,33 @@
 #!/usr/bin/env bash
 
-# Activates the Conda (Python package manager) environment "codegraph" with all packages needed to execute the Jupyter Notebooks.
+# Activates the Conda (Python package manager) environment "codegraph" with all packages needed to run the included Jupyter Notebooks and Python scripts.
 
 # Note: This script uses the conda environment defined in CODEGRAPH_CONDA_ENVIRONMENT (defaults to "codegraph").
-#       If the environment hadn't been created yet, it will use "conda-environment.yml" from the root directory
-#       in the same directory as the given jupyter notebook ipynb file
-#       to create the environment.
+#       If the environment hadn't been created yet, it will use "conda-environment.yml" from the root directory to create the environment.
 
 # Requires operatingSystemFunctions.sh
 
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+PREPARE_CONDA_ENVIRONMENT=${PREPARE_CONDA_ENVIRONMENT:-"true"} # Wether to prepare a Python environment with Conda if needed (default, "true") or use an already prepared Conda environment ("false")
+
+if [ "${PREPARE_CONDA_ENVIRONMENT}" = "false" ]; then
+    echo "activateCondaEnvironment: Skipping activation. An already activated environment and installed dependencies are expected (PREPARE_CONDA_ENVIRONMENT=false)."
+    # "return" needs to be used here instead of "exit".
+    # This script is included in another script by using "source". 
+    # "exit" would end the main script, "return" just ends this sub script.
+    return 0
+fi 
+
+if [ "${USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV}" = "true" ]; then
+    echo "activateCondaEnvironment: Skipping activation. venv will be used instead of conda (USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV=true)."
+    # "return" needs to be used here instead of "exit".
+    # This script is included in another script by using "source". 
+    # "exit" would end the main script, "return" just ends this sub script.
+    return 0
+fi 
+
 ## Get this "scripts" directory if not already set
 # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
@@ -37,16 +53,6 @@ echo "activateCondaEnvironment: CONDA_PREFIX=${CONDA_PREFIX}"
 echo "activateCondaEnvironment: Current conda environment=${CONDA_DEFAULT_ENV}"
 echo "activateCondaEnvironment: Target conda environment=${CODEGRAPH_CONDA_ENVIRONMENT}"
 
-PREPARE_CONDA_ENVIRONMENT=${PREPARE_CONDA_ENVIRONMENT:-"true"} # Wether to prepare a Python environment with Conda if needed (default, "true") or use an already prepared Conda environment ("false")
-
-if [ "${PREPARE_CONDA_ENVIRONMENT}" = "false" ]; then
-    echo "activateCondaEnvironment: Skipping activation. ${PREPARE_CONDA_ENVIRONMENT} is set to false."
-    # "return" needs to be used here instead of "exit".
-    # This script is included in another script by using "source". 
-    # "exit" would end the main script, "return" just ends this sub script.
-    return 0
-fi 
-
 # Include operation system function to for example detect Windows.
 source "${SCRIPTS_DIR}/operatingSystemFunctions.sh"
 

scripts/activatePythonEnvironment.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+# Activates the .venv environment (Python build-in virtual environments) with all packages necessary to run the included Jupyter Notebooks and Python scripts.
+
+# Note: If the environment hadn't been created yet, it will use "requirements.txt" from the root directory to create the environment.
+
+# Requires operatingSystemFunctions.sh
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV=${USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV:-"false"} # Use "venv" for virtual Python environments ("true") or use an already prepared (e.g. conda) environment (default, "false").
+
+if [ "${USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV}" = "false" ]; then
+    echo "activatePythonEnvironment: Skipping activation. An already activated environment and installed dependencies are expected e.g. by using conda (USE_VIRTUAL_PYTHON_ENVIRONMENT_VENV=false)."
+    # "return" needs to be used here instead of "exit".
+    # This script is included in another script by using "source". 
+    # "exit" would end the main script, "return" just ends this sub script.
+    return 0
+fi
+
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+echo "activatePythonEnvironment: SCRIPTS_DIR=${SCRIPTS_DIR}"
+
+# Get the root directory by taking the path of this script and going one directory up.
+ROOT_DIRECTORY=${ROOT_DIRECTORY:-"$(dirname "${SCRIPTS_DIR}")"} # Repository directory containing the Jupyter Notebooks
+echo "activatePythonEnvironment: ROOT_DIRECTORY=${ROOT_DIRECTORY}"
+
+# Get the file name of the environment description file for the conda package and environment manager 
+# that contains all dependencies and their versions.
+PYTHON_ENVIRONMENT_FILE=${PYTHON_ENVIRONMENT_FILE:-"${ROOT_DIRECTORY}/requirements.txt"} # Pip (package manager for Python) environment file path
+if [ ! -f "${PYTHON_ENVIRONMENT_FILE}" ] ; then
+    echo "activatePythonEnvironment: Couldn't find environment file ${PYTHON_ENVIRONMENT_FILE}."
+    exit_failed
+fi
+
+deactivate_conda_if_necessary() {
+    # Include operation system function to for example detect Windows.
+    source "${SCRIPTS_DIR}/operatingSystemFunctions.sh"
+
+    # Determine the path to "conda"  
+    if [ -n "${CONDA}" ]; then
+        if isWindows; then
+            pathToConda="${CONDA}\\Scripts\\" # the trailing backslash character is required
+        else
+            pathToConda="${CONDA}/bin/" # the trailing slash character is required
+        fi
+    else
+        pathToConda=""
+    fi
+    echo "activatePythonEnvironment: pathToConda=${pathToConda} (for deactivation)"
+
+    scriptExtension=$(ifWindows ".bat" "")
+    echo "activatePythonEnvironment: scriptExtension=${scriptExtension}"
+
+    if "${pathToConda}conda" deactivate >/dev/null 2>&1; then
+        # Call "deactivate" a second time to also deactivate the "base" environment
+        "${pathToConda}conda" deactivate >/dev/null 2>&1;
+        echo "activatePythonEnvironment: Conda deactivated"
+    else
+        echo "activatePythonEnvironment: Conda not found. Deactivation skipped"
+    fi
+}
+
+VENV_DIRECTORY=".venv"
+
+# Create the virtual environment if needed
+if [ ! -d "${ROOT_DIRECTORY}/${VENV_DIRECTORY}" ]; then
+    deactivate_conda_if_necessary
+    echo "activatePythonEnvironment: Creating ${VENV_DIRECTORY} environment..."
+    python3 -m venv "${ROOT_DIRECTORY}/${VENV_DIRECTORY}"
+else
+    echo "activatePythonEnvironment: Already created ${VENV_DIRECTORY} environment."
+fi
+
+# Activate the virtual environment if needed
+if [ "${VIRTUAL_ENV}" != "${ROOT_DIRECTORY}/${VENV_DIRECTORY}" ]; then
+    echo "activatePythonEnvironment: Activate ${VENV_DIRECTORY} environment..."
+    source "${ROOT_DIRECTORY}/${VENV_DIRECTORY}/bin/activate"
+else
+    echo "activatePythonEnvironment: Already activated ${VENV_DIRECTORY} environment."
+fi
+
+# Install the virtual environment if needed
+if pip install --dry-run --no-deps --requirement "${PYTHON_ENVIRONMENT_FILE}" 2>/dev/null | grep -q "Would install"; then
+    echo "activatePythonEnvironment: Install environment dependencies..."
+    pip install -q --requirement "${PYTHON_ENVIRONMENT_FILE}"
+else
+    echo "activatePythonEnvironment: Already installed environment dependencies."
+fi
+
+echo "activatePythonEnvironment: Python installation: $(which python)"

scripts/executeJupyterNotebook.sh

@@ -12,8 +12,11 @@
 
 # Note: This script uses the conda environment defined in CODEGRAPH_CONDA_ENVIRONMENT (defaults to "codegraph").
 #       If the environment hadn't been created yet it will use "conda-environment.yml" 
-#       in the same directory as the given jupyter notebook ipynb file
-#       to create the environment.
+#       in the root directory of this repository to create the environment.
+
+# Note: Besides Conda, Python's build-in virtual environment module "venv" is also supported.
+#       If the environment hadn't been created yet it will use "requirements.txt"
+#       in the root directory of this repository to create the environment.
 
 # Requires juypter nbconvert,operatingSystemFunctions.sh
 
@@ -85,8 +88,12 @@ if [ ! -f "${jupyter_notebook_file_path}/.env" ] ; then
     echo "NEO4J_INITIAL_PASSWORD=${NEO4J_INITIAL_PASSWORD}" > "${jupyter_notebook_file_path}/.env"
 fi
 
-# Create and activate (if necessary) Conda environment as defined in environment variable CODEGRAPH_CONDA_ENVIRONMENT (default "codegraph")
-source "${SCRIPTS_DIR}/activateCondaEnvironment.sh"
+# Create and activate (if necessary) a virtual environment (Conda or venv).
+# For Conda, the environment name is taken from the environment variable CODEGRAPH_CONDA_ENVIRONMENT (default "codegraph")
+# and the dependencies are listed in the root directory file "conda-environment.yml".
+# For venv, the dependencies are listed in the root directory file "requirements.txt".
+time source "${SCRIPTS_DIR}/activateCondaEnvironment.sh"
+time source "${SCRIPTS_DIR}/activatePythonEnvironment.sh"
 
 jupyter --version || exit 1
 

scripts/reports/compilations/PythonReports.sh

@@ -22,10 +22,20 @@ echo "PythonReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT
 REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCRIPT_DIR}")}
 echo "PythonReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}"
 
+SCRIPTS_DIR=${SCRIPTS_DIR:-$(dirname -- "${REPORTS_SCRIPT_DIR}")}
+echo "PythonReports: SCRIPTS_DIR=${SCRIPTS_DIR}"
+
 # Get the "domains" directory that contains analysis and report scripts by functionality.
 DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY:-"${REPORTS_SCRIPT_DIR}/../../domains"}
 echo "PythonReports: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}"
 
+# Create and activate (if necessary) a virtual environment (Conda or venv).
+# For Conda, the environment name is taken from the environment variable CODEGRAPH_CONDA_ENVIRONMENT (default "codegraph")
+# and the dependencies are listed in the root directory file "conda-environment.yml".
+# For venv, the dependencies are listed in the root directory file "requirements.txt".
+time source "${SCRIPTS_DIR}/activateCondaEnvironment.sh"
+time source "${SCRIPTS_DIR}/activatePythonEnvironment.sh"
+
 # Run all Python report scripts (filename ending with Csv.sh) in the REPORTS_SCRIPT_DIR and DOMAINS_DIRECTORY directories.
 for directory in "${REPORTS_SCRIPT_DIR}" "${DOMAINS_DIRECTORY}"; do
     if [ ! -d "${directory}" ]; then