Merge branch 'main' into eessi_container_unionfs

Author: boegel

.github/workflows/scripts/only_latest_easystacks.sh

@@ -6,11 +6,13 @@ on:
   workflow_dispatch:
 permissions:
   contents: read # to fetch code (actions/checkout)
-env:
-  EESSI_VERSION: '2023.06'
 jobs:
   check_eb_hooks:
     runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        EESSI_VERSION:
+          - '2023.06'
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -50,7 +52,10 @@ jobs:
                 git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE"
               fi
 
+              # replace <EESSI_VERSION> placeholder (as is also done in install_scripts.sh)
+              sed -i "s/<EESSI_VERSION>/${{matrix.EESSI_VERSION}}/g" "${TEMP_FILE}"
+
               # Compare the hooks to what is shipped in the repository
-              source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
+              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EESSI-extend
               diff "$TEMP_FILE" "$EASYBUILD_HOOKS"

.github/workflows/test-eb-hooks.yml

@@ -17,6 +17,8 @@ jobs:
           - 2xa100 # cc80, supported with (atleast) zen2 CPU
           - 4xa100 # cc80, supported with (atleast) zen2 CPU
           - cc01 # non-existing GPU
+        EESSI_VERSION:
+          - '2023.06'
       fail-fast: false
     steps:
     - name: checkout
@@ -31,6 +33,10 @@ jobs:
           cvmfs_http_proxy: DIRECT
           cvmfs_repositories: software.eessi.io
 
+    - name: Fix EESSI version in init scripts
+      run: |
+        sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/eessi_defaults
+
     - name: test accelerator detection
       run: |
           export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/amd/zen2'
@@ -68,7 +74,7 @@ jobs:
                   match=$(grep "${pattern}" init.out || true)
                   test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1)
 
-                  pattern="Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
+                  pattern="Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
                   echo ">>> checking for lack of pattern '${pattern}' in init output..."
                   match=$(grep "${pattern}" init.out || true)
                   test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1)
@@ -79,19 +85,19 @@ jobs:
                   echo ">>> checking for pattern '${pattern}' in init output..."
                   grep "${pattern}" init.out || (echo "FAILED 1" || exit 1)
 
-                  pattern="Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
+                  pattern="Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
                   echo ">>> checking for lack of pattern '${pattern}' in init output..."
                   match=$(grep "${pattern}" init.out || true)
                   test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1)
 
               else
                   echo ">>> checking for 'accel/nvidia/cc80' in init output..."
                   grep "archdetect found supported accelerator for CPU target x86_64/amd/zen2: accel/nvidia/cc80" init.out || (echo "FAILED 2" && exit 1)
-                  grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
+                  grep "Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/x86_64/amd/zen2/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
               fi
 
               echo ">>> checking last line of init output..."
-              tail -1 init.out | grep "Environment set up to use EESSI (2023.06), have fun!" || (echo "FAILED, full init utput:" && cat init.out && exit 1)
+              tail -1 init.out | grep "Environment set up to use EESSI (${{matrix.EESSI_VERSION}}), have fun!" || (echo "FAILED, full init utput:" && cat init.out && exit 1)
 
               echo "All checks on init output PASSED"
           else
@@ -118,7 +124,7 @@ jobs:
           echo ">>> checking for 'accel/nvidia/cc80' in init output..."
           grep "archdetect found supported accelerator for CPU target x86_64/amd/zen3: accel/nvidia/cc80" init.out || (echo "FAILED 1" && exit 1)
           grep "Using x86_64/amd/zen2 as software subdirectory" init.out || (echo "FAILED 2" && exit 1)
-          grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
-          grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 4" && exit 1)
+          grep "Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/x86_64/amd/zen2/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
+          grep "Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 4" && exit 1)
 
           echo "All checks on init output PASSED"

.github/workflows/tests_archdetect_nvidia_gpu.yml

@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-          - 2023.06
+          - '2023.06'
     steps:
       - name: Check out software-layer repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -68,7 +68,7 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-          - 2023.06
+          - '2023.06'
         EESSI_SOFTWARE_SUBDIR_OVERRIDE:
           - x86_64/amd/zen3
           - x86_64/amd/zen4
@@ -84,6 +84,10 @@ jobs:
           cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
           cvmfs_http_proxy: DIRECT
           cvmfs_repositories: software.eessi.io
+
+      - name: Fix EESSI version in init scripts
+        run: |
+          sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/eessi_defaults
               
       - name: Test for expected variables match between Lmod init script and original bash script
         run: |
@@ -141,7 +145,7 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-          - 2023.06
+          - '2023.06'
         EESSI_SOFTWARE_SUBDIR_OVERRIDE:
           - none
           - x86_64/amd/zen2

.github/workflows/tests_eessi_module.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-          - 2023.06
+          - '2023.06'
         EESSI_SOFTWARE_SUBDIR_OVERRIDE:
           - x86_64/intel/haswell
     steps:
@@ -27,6 +27,12 @@ jobs:
           cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
           cvmfs_http_proxy: DIRECT
           cvmfs_repositories: software.eessi.io
+
+      - name: Fix EESSI version in lmod init scripts
+        run: |
+          for shell in $(ls init/lmod); do
+            sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/lmod/${shell}
+          done
          
       - name: Clone assert.sh script
         run: git clone https://github.com/lehmannro/assert.sh.git

.github/workflows/tests_init_module.yml

@@ -359,7 +359,7 @@ else
     for easystack_file in ${rebuild_easystacks} ${new_easystacks}; do
 
         # make sure that easystack file being picked up is for EESSI version that we're building for...
-        echo "${easystack_file}" | grep -q "^easystacks/${EESSI_VERSION}/"
+        echo "${easystack_file}" | grep -q "^easystacks/$(basename ${EESSI_CVMFS_REPO})/${EESSI_VERSION}/"
         if [ $? -ne 0 ]; then
             fatal_error "Easystack file ${easystack_file} is not intended for EESSI version ${EESSI_VERSION}, giving up!"
         fi

.github/workflows/tests_readme.yml

@@ -1,45 +1,17 @@
-# Software layer
+# Software layer scripts
 
-The software layer of the EESSI project uses [EasyBuild](https://docs.easybuild.io), [Lmod](https://lmod.readthedocs.io) and [archspec](https://archspec.readthedocs.io).
+Scripts for building the EESSI software layer.
 
 See also https://www.eessi.io/docs/software_layer .
 
-## Pilot software stack
-
-You can set up your environment by sourcing the init script:
-
-```
-$ source /cvmfs/software.eessi.io/versions/2023.06/init/bash
-Found EESSI repo @ /cvmfs/software.eessi.io/versions/2023.06!
-Derived subdirectory for software layer: x86_64/intel/haswell
-Using x86_64/intel/haswell subdirectory for software layer
-Initializing Lmod...
-Prepending /cvmfs/software.eessi.io/versions/2023.06/software/x86_64/intel/haswell/modules/all to $MODULEPATH...
-Environment set up to use EESSI (2023.06), have fun!
-[EESSI 2023.06] $
-```
-
-### Accessing EESSI via a container
-
-You need Singularity version 3.7 or newer. Then, simply run
-
-```
-$ ./eessi_container.sh
-```
-Once you get presented the prompt `Singularity>` run the above `source` command.
-
-If you want to build a package for the software repository, simply add the arguments `--access rw`, e.g., full command would be
-
-```
-$ ./eessi_container.sh --access rw
-```
-Note, not all features/arguments listed via `./eessi_container.sh --help` are implemented.
+Easystack files used for specifying which software installations are included in EESSI software layer are in
+[EESSI/software-layer](https://github.com/EESSI/software-layer) repository.
 
 # License
 
 The software in this repository is distributed under the terms of the
 [GNU General Public License v2.0](https://opensource.org/licenses/GPL-2.0).
 
-See [LICENSE](https://github.com/EESSI/software-layer/blob/main/LICENSE) for more information.
+See [LICENSE](https://github.com/EESSI/software-layer-scripts/blob/main/LICENSE) for more information.
 
 SPDX-License-Identifier: GPL-2.0-only

EESSI-install-software.sh

@@ -408,7 +408,7 @@ def parse_hook_zen4_module_only(ec, eprefix):
         # Need to escape newline character so that the newline character actually ends up in the module file
         # (otherwise, it splits the string, and a 2-line string ends up in the modulefile, resulting in syntax error)
         errmsg = "EasyConfigs using toolchains based on GCCcore-12.2.0 are not supported for the Zen4 architecture.\\n"
-        errmsg += "See https://www.eessi.io/docs/known_issues/eessi-2023.06/#gcc-1220-and-foss-2022b-based-modules-cannot-be-loaded-on-zen4-architecture"
+        errmsg += "See https://www.eessi.io/docs/known_issues/eessi-<EESSI_VERSION>/#gcc-1220-and-foss-2022b-based-modules-cannot-be-loaded-on-zen4-architecture"
         ec['modluafooter'] = 'if (not os.getenv("%s")) then LmodError("%s") end' % (env_varname, errmsg)
 
 

README.md

@@ -8,6 +8,8 @@
 # license: GPLv2
 #
 
+export EESSI_VERSION_DEFAULT='__EESSI_VERSION_DEFAULT__'
+
 # use different defaults for RISC-V, as we want to redirect to the riscv.eessi.io repo
 if [[ $(uname -m) == "riscv64" ]]; then
     export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/riscv.eessi.io}"
@@ -20,7 +22,7 @@ if [[ $(uname -m) == "riscv64" ]]; then
     fi
 else
     export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/software.eessi.io}"
-    export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}"
+    export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=${EESSI_VERSION_DEFAULT}}"
 fi
 # use archdetect by default, unless otherwise specified
 export EESSI_USE_ARCHDETECT="${EESSI_USE_ARCHDETECT:=1}"