diff --git a/.github/workflows/scripts/verify_eessi_environment.py b/.github/workflows/scripts/verify_eessi_environment.py index cc006e70..9b580969 100644 --- a/.github/workflows/scripts/verify_eessi_environment.py +++ b/.github/workflows/scripts/verify_eessi_environment.py @@ -45,8 +45,8 @@ def check_env_endswith(var1, var2): check_env_equals("EESSI_ACCELERATOR_TARGET_OVERRIDE", "EESSI_ACCEL_SUBDIR") # special case is where EESSI_ACCELERATOR_TARGET_OVERRIDE may not match the final # accelerator architecture chosen. - # In CI we set FINAL_ACCELERATOR_PATH_EXPECTED to allow us to compare against an expected value. - check_env_equals("EESSI_ACCELERATOR_TARGET", "FINAL_ACCELERATOR_PATH_EXPECTED") + # In CI we set FINAL_ACCELERATOR_TARGET_EXPECTED to allow us to compare against an expected value. + check_env_equals("EESSI_ACCELERATOR_TARGET", "FINAL_ACCELERATOR_TARGET_EXPECTED") # verify the software paths that should exist check_env_endswith("EESSI_SOFTWARE_PATH", "EESSI_SOFTWARE_SUBDIR") check_env_endswith("EESSI_SITE_SOFTWARE_PATH", "EESSI_SOFTWARE_SUBDIR") diff --git a/.github/workflows/tests_eessi_extend_module.yml b/.github/workflows/tests_eessi_extend_module.yml index f304c48b..7d8e9daa 100644 --- a/.github/workflows/tests_eessi_extend_module.yml +++ b/.github/workflows/tests_eessi_extend_module.yml @@ -105,7 +105,8 @@ jobs: export EESSI_PROJECT_INSTALL="$MY_INSTALLATION_PATH" module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # check some specific envvars - check_env_var "EASYBUILD_INSTALLPATH" "$MY_INSTALLATION_PATH/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR" + export EXPECTED_INSTALLATION_PATH="$MY_INSTALLATION_PATH/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR" + check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH" check_env_var "EASYBUILD_UMASK" "002" check_env_var "EASYBUILD_GROUP_WRITABLE_INSTALLDIR" "1" # unload and check the environment is clean again @@ -118,10 +119,92 @@ jobs: mkdir -p $EESSI_USER_INSTALL # must exist module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # check some specific envvars - check_env_var "EASYBUILD_INSTALLPATH" "$MY_INSTALLATION_PATH/$USER/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR" + export EXPECTED_INSTALLATION_PATH="$MY_INSTALLATION_PATH/$USER/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR" + check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH" check_env_var "EASYBUILD_UMASK" "077" # unload and check the environment is clean again module unload EESSI-extend check_disallowed_env_prefix EASYBUILD_ unset EESSI_USER_INSTALL + - name: Run tests for EESSI-extend in the various GPU scenarios + run: | + export MY_INSTALLATION_PATH=/tmp/easybuild + + # Define a function to check the values of environment variables + # and another that checks an environment does not contain environment + # variables matching a certain pattern + source .github/workflows/scripts/test_utils.sh + + # Set an environment variable to use when we want to target accelerators + export STORED_EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc80" + export STORED_CUDA_CC="8.0" + + # Let's start from a clean slate + module purge + export EESSI_ACCELERATOR_TARGET_OVERRIDE=$STORED_EESSI_ACCELERATOR_TARGET_OVERRIDE + module load EESSI/${{matrix.EESSI_VERSION}} + # Access the installed EESSI-extend + module use "$MY_INSTALLATION_PATH"/modules/all + check_disallowed_env_prefix EASYBUILD_ + + # Configure for CVMFS install + export EESSI_CVMFS_INSTALL=1 + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + check_env_var "EASYBUILD_INSTALLPATH" "$EESSI_SOFTWARE_PATH" # installation path should be the same unless we ask for an explicit GPU installation + check_env_var "EASYBUILD_CUDA_COMPUTE_CAPABILITIES" "$STORED_CUDA_CC" + export EESSI_ACCELERATOR_INSTALL=1 + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # reload for an actual GPU installation + check_env_var "EASYBUILD_INSTALLPATH" "${EESSI_SOFTWARE_PATH}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}" + # unload and make sure the environment is clean again + module unload EESSI-extend + check_disallowed_env_prefix EASYBUILD_ + unset EESSI_ACCELERATOR_INSTALL + unset EESSI_CVMFS_INSTALL + + # Now configure for a site + export EESSI_SITE_INSTALL=1 + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + check_env_var "EASYBUILD_INSTALLPATH" "$EESSI_SITE_SOFTWARE_PATH" # installation path should be the same unless we ask for an explicit GPU installation + check_env_var "EASYBUILD_CUDA_COMPUTE_CAPABILITIES" "$STORED_CUDA_CC" + export EESSI_ACCELERATOR_INSTALL=1 + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # reload for an actual GPU installation + check_env_var "EASYBUILD_INSTALLPATH" "${EESSI_SITE_SOFTWARE_PATH}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}" + # unload and make sure the environment is clean again + module unload EESSI-extend + check_disallowed_env_prefix EASYBUILD_ + unset EESSI_ACCELERATOR_INSTALL + unset EESSI_SITE_INSTALL + + # Now for a project + export EESSI_PROJECT_INSTALL="$MY_INSTALLATION_PATH" + export EXPECTED_INSTALLATION_PATH="$MY_INSTALLATION_PATH/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR" + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH" # installation path should be the same unless we ask for an explicit GPU installation + check_env_var "EASYBUILD_CUDA_COMPUTE_CAPABILITIES" "$STORED_CUDA_CC" + export EESSI_ACCELERATOR_INSTALL=1 + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # reload for an GPU actual installation + check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH" # installation path should be the same for project case + # unload and make sure the environment is clean again + module unload EESSI-extend + check_disallowed_env_prefix EASYBUILD_ + unset EESSI_ACCELERATOR_INSTALL + unset EESSI_PROJECT_INSTALL + + # Now for a user + export EESSI_USER_INSTALL="$MY_INSTALLATION_PATH/$USER" + mkdir -p $EESSI_USER_INSTALL # must exist + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + # check some specific envvars + export EXPECTED_INSTALLATION_PATH="$MY_INSTALLATION_PATH/$USER/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR" + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild + check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH" # installation path should be the same unless we ask for an explicit GPU installation + check_env_var "EASYBUILD_CUDA_COMPUTE_CAPABILITIES" "$STORED_CUDA_CC" + export EESSI_ACCELERATOR_INSTALL=1 + module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # reload for an actual GPU installation + check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH" # installation path should be the same for user case + # unload and make sure the environment is clean again + module unload EESSI-extend + check_disallowed_env_prefix EASYBUILD_ + unset EESSI_ACCELERATOR_INSTALL + unset EESSI_USER_INSTALL diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index d6c97a40..c13b182b 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -165,9 +165,9 @@ jobs: include: # For each override we expect a specific path (which may differ from the original due to overrides) - EESSI_ACCELERATOR_TARGET_OVERRIDE: accel/nvidia/cc80 - FINAL_ACCELERATOR_PATH_EXPECTED: accel/nvidia/cc80 + FINAL_ACCELERATOR_TARGET_EXPECTED: accel/nvidia/cc80 - EESSI_ACCELERATOR_TARGET_OVERRIDE: accel/nvidia/cc77 # deliberately chose a non-existent CUDA capability - FINAL_ACCELERATOR_PATH_EXPECTED: accel/nvidia/cc70 # this reverts to the fallback case (which does exist) + FINAL_ACCELERATOR_TARGET_EXPECTED: accel/nvidia/cc70 # this reverts to the fallback case (which does exist) steps: - name: Check out software-layer repository @@ -193,7 +193,7 @@ jobs: # Set our accelerator path overrides according to our matrix if [[ "${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}}" != "none" ]]; then export EESSI_ACCELERATOR_TARGET_OVERRIDE=${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}} - export FINAL_ACCELERATOR_PATH_EXPECTED=${{matrix.FINAL_ACCELERATOR_PATH_EXPECTED}} + export FINAL_ACCELERATOR_TARGET_EXPECTED=${{matrix.FINAL_ACCELERATOR_TARGET_EXPECTED}} fi # Turn on debug output in case we want to take a look diff --git a/EESSI-extend-easybuild.eb b/EESSI-extend-easybuild.eb index 36881252..b0e2364a 100644 --- a/EESSI-extend-easybuild.eb +++ b/EESSI-extend-easybuild.eb @@ -35,6 +35,11 @@ description = """ If both EESSI_USER_INSTALL and EESSI_PROJECT_INSTALL are defined, both sets of installations are exposed, but new installations are created as user installations. + + Strict installation path checking is enforced by EESSI for EESSI and site + installations involving accelerators. In these cases, if you wish to create an + accelerator installation you must set the environment variable + EESSI_ACCELERATOR_INSTALL (and load/reload this module). """ toolchain = SYSTEM @@ -78,8 +83,21 @@ if (mode() == "load") then end end working_dir = os.getenv("WORKING_DIR") or pathJoin("/tmp", os.getenv("USER")) + -- Gather the EPREFIX to use as a sysroot sysroot = os.getenv("EESSI_EPREFIX") + +-- Check if we have GPU capabilities and configure CUDA compute capabilities +eessi_accelerator_target = os.getenv("EESSI_ACCELERATOR_TARGET") +if (eessi_accelerator_target ~= nil) then + cuda_compute_capability = string.match(eessi_accelerator_target, "^accel/nvidia/cc([0-9][0-9])$") + if (cuda_compute_capability ~= nil) then + easybuild_cuda_compute_capabilities = cuda_compute_capability:sub(1, 1) .. "." .. cuda_compute_capability:sub(2, 2) + else + LmodError("Incorrect value for $EESSI_ACCELERATOR_TARGET: " .. eessi_accelerator_target) + end +end + -- Use an installation prefix that we _should_ have write access to if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then -- Make sure no other EESSI install environment variables are set @@ -88,15 +106,9 @@ if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then end eessi_cvmfs_install = true easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH") - eessi_accelerator_target = os.getenv("EESSI_ACCELERATOR_TARGET") - if (eessi_accelerator_target ~= nil) then - cuda_compute_capability = string.match(eessi_accelerator_target, "^nvidia/cc([0-9][0-9])$") - if (cuda_compute_capability ~= nil) then - easybuild_installpath = pathJoin(easybuild_installpath, 'accel', eessi_accelerator_target) - easybuild_cuda_compute_capabilities = cuda_compute_capability:sub(1, 1) .. "." .. cuda_compute_capability:sub(2, 2) - else - LmodError("Incorrect value for $EESSI_ACCELERATOR_TARGET: " .. eessi_accelerator_target) - end + -- enforce accelerator subdirectory usage for CVMFS installs (only if an accelerator install is requested) + if (eessi_accelerator_target ~= nil) and (cuda_compute_capability ~= nil) and (os.getenv("EESSI_ACCELERATOR_INSTALL") ~= nil) then + easybuild_installpath = pathJoin(easybuild_installpath, eessi_accelerator_target) end elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then -- Make sure no other EESSI install environment variables are set @@ -104,6 +116,10 @@ elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables") end easybuild_installpath = os.getenv("EESSI_SITE_SOFTWARE_PATH") + -- enforce accelerator subdirectory usage for site installs (only if an accelerator install is requested) + if (eessi_accelerator_target ~= nil) and (cuda_compute_capability ~= nil) and (os.getenv("EESSI_ACCELERATOR_INSTALL") ~= nil) then + easybuild_installpath = pathJoin(easybuild_installpath, eessi_accelerator_target) + end else -- Deal with user and project installs project_install = os.getenv("EESSI_PROJECT_INSTALL") diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index c491819f..2d5a62c7 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -284,6 +284,16 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi +# If we have EESSI_ACCELERATOR_TARGET_OVERRIDE set (and non-empty), then this implies building for a GPU target +# (this must be set _before_ we load EESSI-extend). +# We also make sure that EESSI_ACCELERATOR_TARGET is also set as EESSI_ACCELERATOR_TARGET_OVERRIDE must +# be set before the EESSI module is loaded in order to set accelerator information. +if [[ -n "$EESSI_ACCELERATOR_TARGET_OVERRIDE" && -z "$EESSI_ACCELERATOR_TARGET" ]]; then + fatal_error "EESSI module should've set EESSI_ACCELERATOR_TARGET ($EESSI_ACCELERATOR_TARGET) when EESSI_ACCELERATOR_TARGET_OVERRIDE ($EESSI_ACCELERATOR_TARGET_OVERRIDE) exported." +elif [[ -n "$EESSI_ACCELERATOR_TARGET_OVERRIDE" ]]; then + export EESSI_ACCELERATOR_INSTALL=1 +fi + echo "DEBUG: before loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'" source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} echo "DEBUG: after loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'" diff --git a/bot/build.sh b/bot/build.sh index 2bba0cba..45a15f23 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -162,8 +162,14 @@ export EESSI_SOFTWARE_SUBDIR_OVERRIDE echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" # determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} -export EESSI_ACCELERATOR_TARGET=$(cfg_get_value "architecture" "accelerator") -echo "bot/build.sh: EESSI_ACCELERATOR_TARGET='${EESSI_ACCELERATOR_TARGET}'" +ACCEL_OVERRIDE=$(cfg_get_value "architecture" "accelerator") +if [[ -n "$ACCEL_OVERRIDE" ]]; then + # bot job config does not include accel subdirectory + export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/$ACCEL_OVERRIDE" +else + export EESSI_ACCELERATOR_TARGET_OVERRIDE="" +fi +echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${EESSI_ACCELERATOR_TARGET_OVERRIDE}'" # get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index e9a49cbe..188420e6 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -29,8 +29,8 @@ fi if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi -if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then - INPUT="export EESSI_ACCELERATOR_TARGET=${EESSI_ACCELERATOR_TARGET}; ${INPUT}" +if [ ! -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]; then + INPUT="export EESSI_ACCELERATOR_TARGET_OVERRIDE=${EESSI_ACCELERATOR_TARGET_OVERRIDE}; ${INPUT}" fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}"