From cba4afc0b236d8a31a33f65f8a40faa0b88b3ddf Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 17 Jun 2025 14:33:31 +0200 Subject: [PATCH 1/4] Add CI for EESSI-extend, and increase it's capabilities --- .github/workflows/scripts/test_utils.sh | 47 +++++++++++++ .../workflows/tests_eessi_extend_module.yml | 70 +++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 .github/workflows/scripts/test_utils.sh create mode 100644 .github/workflows/tests_eessi_extend_module.yml diff --git a/.github/workflows/scripts/test_utils.sh b/.github/workflows/scripts/test_utils.sh new file mode 100644 index 00000000..66d47460 --- /dev/null +++ b/.github/workflows/scripts/test_utils.sh @@ -0,0 +1,47 @@ +#!/bin/bash +check_env_var() { + # Expected usage: check_env_var "MY_ENV_VAR" "foo" + var_name="$1" + expected="$2" + actual="${!var_name}" + if [ "$actual" != "$expected" ]; then + echo "ERROR: $var_name is '$actual', expected '$expected'" >&2 + exit 1 + else + echo "$var_name is correctly set to '$expected'" + fi +} + +check_disallowed_env_prefix() { + prefix="$1" + shift + whitelist=("$@") + + disallowed=() + + while IFS='=' read -r var _; do + if [[ "$var" == "$prefix"* ]]; then + allowed=false + for allowed_var in "${whitelist[@]}"; do + if [[ "$var" == "$allowed_var" ]]; then + allowed=true + break + fi + done + + if ! $allowed; then + disallowed+=("$var") + fi + fi + done < <(env) + + if [ "${#disallowed[@]}" -ne 0 ]; then + echo "ERROR: Found disallowed environment variables with prefix '$prefix':" >&2 + for var in "${disallowed[@]}"; do + echo " - $var" >&2 + done + exit 1 + else + echo "✅ No disallowed environment variables with prefix '$prefix' found." + fi +} diff --git a/.github/workflows/tests_eessi_extend_module.yml b/.github/workflows/tests_eessi_extend_module.yml new file mode 100644 index 00000000..38605c2b --- /dev/null +++ b/.github/workflows/tests_eessi_extend_module.yml @@ -0,0 +1,70 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for EESSI-extend module functionality in software.eessi.io +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + basic_checks: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - '2023.06' + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Mount EESSI CernVM-FS repository + uses: eessi/github-action-eessi@v3 + with: + eessi_stack_version: ${{matrix.EESSI_VERSION}} + + - name: Install the EESSI-extend shipped with the repository + run: | + # EESSI is already available + module load EasyBuild + # Set an installation path + export MY_INSTALLATION_PATH=/tmp/easybuild + export EASYBUILD_PREFIX=$MY_INSTALLATION_PATH + eb EESSI-extend-easybuild.eb --rebuild + # Verify that we can pick the installed version up + module use $MY_INSTALLATION_PATH/modules/all + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + echo $EBROOTEESSIMINEXTEND | grep $MY_INSTALLATION_PATH || { echo "ERROR: Installed version of EESSI-extend not picked up (loaded $EBROOTEESSIMINEXTEND)" >&2; exit 1; } + module unload EESSI-extend + module unuse $MY_INSTALLATION_PATH/modules/all + unset EASYBUILD_PREFIX + + - name: Run tests for EESSI-extend in the various scenarios + run: | + export MY_INSTALLATION_PATH=/tmp/easybuild + + # Define a function to check the values of environment variables + # and another that checks an environment does not contain environment + # variables matching a certain pattern + source .github/workflows/scripts/test_utils.sh + + # Let's start from a clean slate + module purge + check_disallowed_env_prefix EESSI_ + check_disallowed_env_prefix EASYBUILD_ + + # Load the EESSI module + module load EESSI/${{matrix.EESSI_VERSION}} + check_disallowed_env_prefix EASYBUILD_ + # Access the installed EESSI-extend + module use /tmp/easybuild/modules/all + + # Configure for CVMFS install + export EESSI_CVMFS_INSTALL=1 + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + # check some common ones + check_env_var "EASYBUILD_READ_ONLY_INSTALLDIR" "1" + check_env_var "EASYBUILD_ALLOW_LOADED_MODULES" "EasyBuild,EESSI-extend" + check_env_var "EASYBUILD_SYSROOT" "$EESSI_EPREFIX" + # check some specific ones + check_env_var "EASYBUILD_INSTALLPATH" "$EESSI_SOFTWARE_PATH" From ae35c9b36159c96e0973a377792467cd6b8cb056 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 17 Jun 2025 15:27:28 +0200 Subject: [PATCH 2/4] Use Lmod initialisation for more flexibility --- .github/workflows/tests_eessi_extend_module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_eessi_extend_module.yml b/.github/workflows/tests_eessi_extend_module.yml index 38605c2b..9dc58f33 100644 --- a/.github/workflows/tests_eessi_extend_module.yml +++ b/.github/workflows/tests_eessi_extend_module.yml @@ -22,6 +22,7 @@ jobs: uses: eessi/github-action-eessi@v3 with: eessi_stack_version: ${{matrix.EESSI_VERSION}} + use_eessi_module: true - name: Install the EESSI-extend shipped with the repository run: | From 9d0dac6e8309a8fdcc0622e8e4b8dd1a40d1a4d5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 17 Jun 2025 16:09:06 +0200 Subject: [PATCH 3/4] Extend CI for the other scenarios --- .../workflows/tests_eessi_extend_module.yml | 73 ++++++++++++++++--- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests_eessi_extend_module.yml b/.github/workflows/tests_eessi_extend_module.yml index 9dc58f33..528efffd 100644 --- a/.github/workflows/tests_eessi_extend_module.yml +++ b/.github/workflows/tests_eessi_extend_module.yml @@ -26,21 +26,32 @@ jobs: - name: Install the EESSI-extend shipped with the repository run: | - # EESSI is already available + # Let's start from a clean slate (unload the EESSI module) + module purge + check_disallowed_env_prefix EESSI_ + check_disallowed_env_prefix EASYBUILD_ + + # Load the EESSI module + module load EESSI/${{matrix.EESSI_VERSION}} + check_disallowed_env_prefix EASYBUILD_ + + # Load EasyBuild to install EESSI-extend module load EasyBuild # Set an installation path export MY_INSTALLATION_PATH=/tmp/easybuild export EASYBUILD_PREFIX=$MY_INSTALLATION_PATH eb EESSI-extend-easybuild.eb --rebuild + # Verify that we can pick the installed version up module use $MY_INSTALLATION_PATH/modules/all module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild echo $EBROOTEESSIMINEXTEND | grep $MY_INSTALLATION_PATH || { echo "ERROR: Installed version of EESSI-extend not picked up (loaded $EBROOTEESSIMINEXTEND)" >&2; exit 1; } module unload EESSI-extend + # That should have unset all EasyBuild envvars (including EASYBUILD_PREFIX) + check_disallowed_env_prefix EASYBUILD_ module unuse $MY_INSTALLATION_PATH/modules/all - unset EASYBUILD_PREFIX - - - name: Run tests for EESSI-extend in the various scenarios + + - name: Run tests for EESSI-extend in the various CPU-only scenarios run: | export MY_INSTALLATION_PATH=/tmp/easybuild @@ -51,21 +62,61 @@ jobs: # Let's start from a clean slate module purge - check_disallowed_env_prefix EESSI_ - check_disallowed_env_prefix EASYBUILD_ - - # Load the EESSI module module load EESSI/${{matrix.EESSI_VERSION}} - check_disallowed_env_prefix EASYBUILD_ # Access the installed EESSI-extend - module use /tmp/easybuild/modules/all + module use "$MY_INSTALLATION_PATH"/modules/all + check_disallowed_env_prefix EASYBUILD_ # Configure for CVMFS install export EESSI_CVMFS_INSTALL=1 module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild - # check some common ones + # check some common EasyBuild settings check_env_var "EASYBUILD_READ_ONLY_INSTALLDIR" "1" check_env_var "EASYBUILD_ALLOW_LOADED_MODULES" "EasyBuild,EESSI-extend" check_env_var "EASYBUILD_SYSROOT" "$EESSI_EPREFIX" + check_env_var "EASYBUILD_HOOKS" "/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/easybuild/eb_hooks.py" + check_env_var "EASYBUILD_RPATH" "1" + check_env_var "EASYBUILD_FILTER_ENV_VARS" "LD_LIBRARY_PATH" # check some specific ones check_env_var "EASYBUILD_INSTALLPATH" "$EESSI_SOFTWARE_PATH" + check_env_var "EASYBUILD_UMASK" "022" + # unload and check the environment is clean again + module unload EESSI-extend + check_disallowed_env_prefix EASYBUILD_ + unset EESSI_CVMFS_INSTALL + + # Now configure for a site + export EESSI_SITE_INSTALL=1 + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + # check some specific envvars + check_env_var "EASYBUILD_INSTALLPATH" "$EESSI_SITE_SOFTWARE_PATH" + check_env_var "EASYBUILD_UMASK" "022" + # unload and check the environment is clean again + module unload EESSI-extend + check_disallowed_env_prefix EASYBUILD_ + unset EESSI_SITE_INSTALL + + # Now for a project + export EESSI_PROJECT_INSTALL="$MY_INSTALLATION_PATH" + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + # check some specific envvars + check_env_var "EASYBUILD_INSTALLPATH" "$MY_INSTALLATION_PATH/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR" + check_env_var "EASYBUILD_UMASK" "002" + check_env_var "EASYBUILD_GROUP_WRITABLE_INSTALLDIR" "1" + # unload and check the environment is clean again + module unload EESSI-extend + check_disallowed_env_prefix EASYBUILD_ + unset EESSI_PROJECT_INSTALL + + # Now for a user + export EESSI_USER_INSTALL="$MY_INSTALLATION_PATH/$USER" + mkdir -p $EESSI_USER_INSTALL # must exist + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + # check some specific envvars + check_env_var "EASYBUILD_INSTALLPATH" "$MY_INSTALLATION_PATH/$USER/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR" + check_env_var "EASYBUILD_UMASK" "077" + # unload and check the environment is clean again + module unload EESSI-extend + check_disallowed_env_prefix EASYBUILD_ + unset EESSI_USER_INSTALL + From b7dc8df89d8f62b9ee00a1a457bd765cfa4e2080 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 17 Jun 2025 16:12:44 +0200 Subject: [PATCH 4/4] Forgot a source --- .github/workflows/tests_eessi_extend_module.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests_eessi_extend_module.yml b/.github/workflows/tests_eessi_extend_module.yml index 528efffd..ef66bcf0 100644 --- a/.github/workflows/tests_eessi_extend_module.yml +++ b/.github/workflows/tests_eessi_extend_module.yml @@ -26,6 +26,11 @@ jobs: - name: Install the EESSI-extend shipped with the repository run: | + # Define a function to check the values of environment variables + # and another that checks an environment does not contain environment + # variables matching a certain pattern + source .github/workflows/scripts/test_utils.sh + # Let's start from a clean slate (unload the EESSI module) module purge check_disallowed_env_prefix EESSI_