Use EasyBuild hooks to limit toolchains possible for specific EESSI version

.github/workflows/scripts/test_init_scripts.sh

@@ -23,7 +23,7 @@ for shell in ${SHELLS[@]}; do
 		echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m" 
   else
 		# TEST 1: Source Script and check Module Output
-		assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully"
+		assert "$shell -c 'source init/lmod/$shell' 2>&1 " "Module for EESSI/$EESSI_VERSION loaded successfully"
 		# TEST 2: Check if module overviews first section is the loaded EESSI module
 		MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
 		PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"

.github/workflows/test-eb-hooks.yml

@@ -1,40 +1,32 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
-name: Check whether eb_hooks.py script is up-to-date
+name: Run checks on EasyBuild hooks script
 on:
   push:
   pull_request:
   workflow_dispatch:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
-  check_eb_hooks:
+  check_eb_hooks_uptodate:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
         EESSI_VERSION:
           - '2023.06'
+          - '2025.06'
+
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
           with:
               fetch-depth: 0  # Fetch all history for all branches and tags
-
-        - name: Show host system info
-          run: |
-            echo "/proc/cpuinfo:"
-            cat /proc/cpuinfo
-            echo
-            echo "lscpu:"
-            lscpu
-
-        - name: Mount EESSI CernVM-FS pilot repository
-          uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+        - name: Mount EESSI CernVM-FS repository
+          uses: eessi/github-action-eessi@v3
           with:
-              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
-              cvmfs_http_proxy: DIRECT
-              cvmfs_repositories: software.eessi.io
+            eessi_stack_version: ${{matrix.EESSI_VERSION}}
+            use_eessi_module: true
 
-        - name: Check that EasyBuild hook is up to date
+        - name: Check whether eb_hooks.py script is up-to-date
           if: ${{ github.event_name == 'pull_request' }}
           run: |
               FILE="eb_hooks.py"
@@ -56,6 +48,71 @@ jobs:
               sed -i "s/<EESSI_VERSION>/${{matrix.EESSI_VERSION}}/g" "${TEMP_FILE}"
 
               # Compare the hooks to what is shipped in the repository
-              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EESSI-extend
               diff "$TEMP_FILE" "$EASYBUILD_HOOKS"
+
+  check_eb_hooks_functionality:
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        EESSI_VERSION:
+          - '2023.06'
+          - '2025.06'
+        include:
+          # For each EESSI version we need to test different modules
+          - EESSI_VERSION: '2023.06'
+            COMPATIBLE_EASYCONFIG: 'M4-1.4.19-GCCcore-13.2.0.eb'
+            INCOMPATIBLE_EASYCONFIG: 'M4-1.4.19-GCCcore-14.2.0.eb'
+            # Pick a site toolchain that will allow the incompatible easyconfig
+            # (the name will be modified when exported)
+            SITE_TOP_LEVEL_TOOLCHAINS: '[{"name": "GCCcore", "version": "14.2.0"}]'
+          - EESSI_VERSION: '2025.06'
+            COMPATIBLE_EASYCONFIG: 'M4-1.4.19-GCCcore-14.2.0.eb'
+            INCOMPATIBLE_EASYCONFIG: 'M4-1.4.19-GCCcore-13.2.0.eb'
+            # Pick a site toolchain that will allow the incompatible easyconfig
+            # (the name will be modified when exported)
+            SITE_TOP_LEVEL_TOOLCHAINS: '[{"name": "GCCcore", "version": "13.2.0"}]'
+
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - name: Mount EESSI CernVM-FS repository
+          uses: eessi/github-action-eessi@v3
+          with:
+            eessi_stack_version: ${{matrix.EESSI_VERSION}}
+            use_eessi_module: true
+
+        - name: Test that hook toolchain verification check works
+          if: ${{ github.event_name == 'pull_request' }}
+          run: |
+              # Set up some environment variables
+              export COMPATIBLE_EASYCONFIG=${{matrix.COMPATIBLE_EASYCONFIG}}
+              export INCOMPATIBLE_EASYCONFIG=${{matrix.INCOMPATIBLE_EASYCONFIG}}
+
+              # Load specific EESSI-extend vertsion (proxies a version check)
+              module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild
+
+              # Test an easyconfig that should work
+              eb --hooks=$PWD/eb_hooks.py "$COMPATIBLE_EASYCONFIG" --stop fetch
+              echo "Success for hook with easyconfig $COMPATIBLE_EASYCONFIG with EESSI/${{matrix.EESSI_VERSION}}"
+
+              # Now ensure an incompatible easyconfig does not work
+              eb --hooks=$PWD/eb_hooks.py "$INCOMPATIBLE_EASYCONFIG" --stop fetch 2>&1 1>/dev/null | grep -q "not supported in EESSI"
+              echo "Found expected failure for hook with easyconfig $INCOMPATIBLE_EASYCONFIG and EESSI/${{matrix.EESSI_VERSION}}"
+
+              # Check the override works
+              EESSI_OVERRIDE_TOOLCHAIN_CHECK=1 eb --hooks=$PWD/eb_hooks.py "$INCOMPATIBLE_EASYCONFIG" --stop fetch
+              echo "Hook ignored via EESSI_OVERRIDE_TOOLCHAIN_CHECK with easyconfig $INCOMPATIBLE_EASYCONFIG and EESSI/${{matrix.EESSI_VERSION}}"
+
+              # Now check if we can set a site list of supported toolchains
+              export SANITIZED_EESSI_VERSION=$(echo "${{ matrix.EESSI_VERSION }}" | sed 's/\./_/g')
+              export EESSI_SITE_TOP_LEVEL_TOOLCHAINS_"$SANITIZED_EESSI_VERSION"='${{matrix.SITE_TOP_LEVEL_TOOLCHAINS}}'
+              eb --hooks=$PWD/eb_hooks.py "$INCOMPATIBLE_EASYCONFIG" --stop fetch
+              echo "Site supported toolchain from $EESSI_SITE_TOP_LEVEL_TOOLCHAINS successfully used with easyconfig $INCOMPATIBLE_EASYCONFIG and EESSI/${{matrix.EESSI_VERSION}}"
+
+              # Make sure an invalid list of dicts fails
+              export EESSI_SITE_TOP_LEVEL_TOOLCHAINS_"$SANITIZED_EESSI_VERSION"="Not a list of dicts"
+              eb --hooks=$PWD/eb_hooks.py "$INCOMPATIBLE_EASYCONFIG" --stop fetch 2>&1 1>/dev/null | grep -q "does not contain a valid list of dictionaries"
+              echo "Incorrect format for EESSI_SITE_TOP_LEVEL_TOOLCHAINS caught"
+

eb_hooks.py

@@ -1,20 +1,24 @@
 # Hooks to customize how EasyBuild installs software in EESSI
 # see https://docs.easybuild.io/en/latest/Hooks.html
+import ast
 import datetime
 import glob
+import json
 import os
 import re
 
 import easybuild.tools.environment as env
 from easybuild.easyblocks.generic.configuremake import obtain_config_guess
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
+from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy
 from easybuild.tools import config
 from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
 from easybuild.tools.config import build_option, install_path, update_build_option
 from easybuild.tools.filetools import apply_regex_substitutions, copy_dir, copy_file, remove_file, symlink, which
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
+from easybuild.tools.toolchain.toolchain import is_system_toolchain
 from easybuild.tools.version import VERSION as EASYBUILD_VERSION
 from easybuild.tools.modules import get_software_root_env_var_name
 
@@ -50,6 +54,18 @@
 
 STACK_REPROD_SUBDIR = 'reprod'
 
+EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS = {
+    '2023.06': [
+        {'name': 'foss', 'version': '2022b'},
+        {'name': 'foss', 'version': '2023a'},
+        {'name': 'foss', 'version': '2023b'},
+    ],
+    '2025.06': [
+        {'name': 'foss', 'version': '2024a'},
+        {'name': 'foss', 'version': '2025a'},
+    ],
+}
+
 
 def is_gcccore_1220_based(**kwargs):
 # ecname, ecversion, tcname, tcversion):
@@ -128,14 +144,73 @@ def parse_hook(ec, *args, **kwargs):
     ec = inject_gpu_property(ec)
 
 
+def parse_list_of_dicts_env(var_name):
+    """Parse a list of dicts that are stored in an environment variable string"""
+
+    # Check if the environment variable name is valid (letters, numbers, underscores, and doesn't start with a digit)
+    if not re.match(r'^[A-Za-z_][A-Za-z0-9_]*$', var_name):
+        raise ValueError(f"Invalid environment variable name: {var_name}")
+    list_string = os.getenv(var_name, '[]')
+
+    list_of_dicts = []
+    try:
+        # Try JSON format first
+        list_of_dicts = json.loads(list_string)
+    except json.JSONDecodeError:
+        try:
+            # Fall back to Python literal format
+            list_of_dicts = ast.literal_eval(list_string)
+        except (ValueError, SyntaxError):
+            raise ValueError(f"Environment variable '{var_name}' does not contain a valid list of dictionaries.")
+
+    return list_of_dicts
+
+
+def verify_toolchains_supported_by_eessi_version(easyconfigs):
+    """Each EESSI version supports a limited set of toolchains, sanity check the easyconfigs for toolchain support."""
+    eessi_version = get_eessi_envvar('EESSI_VERSION')
+    supported_eessi_toolchains = []
+    # Environment variable can't have a '.' so replace by '_'
+    site_top_level_toolchains_envvar = 'EESSI_SITE_TOP_LEVEL_TOOLCHAINS_' + eessi_version.replace('.', '_')
+    site_top_level_toolchains = parse_list_of_dicts_env(site_top_level_toolchains_envvar)
+    for top_level_toolchain in EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS[eessi_version] + site_top_level_toolchains:
+        supported_eessi_toolchains += get_toolchain_hierarchy(top_level_toolchain)
+    for ec in easyconfigs:
+        toolchain = ec['ec']['toolchain']
+        # if it is a system toolchain or appears in the list, we are all good
+        if is_system_toolchain(toolchain['name']):
+            continue
+        # This check verifies that the toolchain dict is in the list of supported toolchains.
+        # It uses <= as there may be other dict entries in the values returned from get_toolchain_hierarchy()
+        # but we only care that the toolchain dict (which has 'name' and 'version') appear.
+        elif not any(toolchain.items() <= supported.items() for supported in supported_eessi_toolchains):
+            raise EasyBuildError(
+                f"Toolchain {toolchain} (required by {ec['full_mod_name']}) is not supported in EESSI/{eessi_version}\n"
+                f"Supported toolchains are:\n" + "\n".join(sorted("  " + str(tc) for tc in supported_eessi_toolchains))
+            )
+
+
+def pre_build_and_install_loop_hook(easyconfigs):
+    """Main pre_build_and_install_loop hook: trigger custom functions before beginning installation loop."""
+
+    # Always check that toolchain supported by the EESSI version (unless overridden)
+    if os.getenv("EESSI_OVERRIDE_TOOLCHAIN_CHECK"):
+        print_warning("Overriding the check that the toolchains are supported by the EESSI version.")
+    else:
+        verify_toolchains_supported_by_eessi_version(easyconfigs)
+
+
 def post_ready_hook(self, *args, **kwargs):
     """
     Post-ready hook: limit parallellism for selected builds based on software name and CPU target.
                      parallelism needs to be limited because some builds require a lot of memory per used core.
     """
     # 'parallel' easyconfig parameter (EB4) or the parallel property (EB5) is set via EasyBlock.set_parallel
     # in ready step based on available cores
-    parallel = getattr(self, 'parallel', self.cfg['parallel'])
+    if hasattr(self, 'parallel'):
+        parallel = self.parallel
+    else:
+        parallel = self.cfg['parallel']
 
     if parallel == 1:
         return  # no need to limit if already using 1 core
@@ -167,7 +242,7 @@ def post_ready_hook(self, *args, **kwargs):
 
     # apply the limit if it's different from current
     if new_parallel != parallel:
-        if EASYBUILD_VERSION >= '5':
+        if hasattr(self, 'parallel'):
             self.cfg.parallel = new_parallel
         else:
             self.cfg['parallel'] = new_parallel
@@ -394,7 +469,7 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs):
     https://github.com/EESSI/software-layer/pull/736#issuecomment-2373261889
     """
     if ec.name == 'FreeImage' and ec.version in ('3.18.0',):
-        if os.getenv('EESSI_CPU_FAMILY') == 'aarch64':
+        if get_eessi_envvar('EESSI_CPU_FAMILY') == 'aarch64':
             # Make sure the toolchainopts key exists, and the value is a dict,
             # before we add the option to enable PIC and disable PNG_ARM_NEON_OPT
             if 'toolchainopts' not in ec or ec['toolchainopts'] is None:
@@ -1230,7 +1305,7 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, pkg_name, al
                     # CUDA and cu* libraries themselves don't care about compute capability so remove this
                     # duplication from under host_injections (symlink to a single CUDA or cu* library
                     # installation for all compute capabilities)
-                    accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET")
+                    accel_subdir = get_eessi_envvar("EESSI_ACCELERATOR_TARGET")
                     if accel_subdir:
                         host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '')
                     # make sure source and target of symlink are not the same
@@ -1326,7 +1401,7 @@ def post_easyblock_hook(self, *args, **kwargs):
 
         # Always trigger this one for EESSI CVMFS/site installations and version 2025.06 or newer, regardless of self.name
         if os.getenv('EESSI_CVMFS_INSTALL') or os.getenv('EESSI_SITE_INSTALL'):
-            if os.getenv('EESSI_VERSION') and LooseVersion(os.getenv('EESSI_VERSION')) >= '2025.06':
+            if get_eessi_envvar('EESSI_VERSION') and LooseVersion(get_eessi_envvar('EESSI_VERSION')) >= '2025.06':
                 post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs)
         else:
             self.log.debug("No CVMFS/site installation requested, not running post_easyblock_hook_copy_easybuild_subdir.")