update Lmod cache for NESSI/2022.11

.github/workflows/test_eessi.yml

@@ -40,4 +40,33 @@ jobs:
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
+              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
+
+        - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
+          run: |
+              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              module load EasyBuild
+              eb --version
+              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              env | grep ^EESSI | sort
+              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)"
+              echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
+              # note, check_missing_installations.sh exits 1 if a package was
+              # missing, which is intepreted as false (exit code based, not
+              # boolean logic), hence when the script exits 0 if no package was
+              # missing it is interpreted as true, thus the test did not capture
+              # the missing package
+              if ./check_missing_installations.sh; then
+                echo "did NOT capture missing package; test FAILED"
+                exit 1
+              else
+                echo "captured missing package; test PASSED"
+                exit 0
+              fi

.github/workflows/test_eessi_container_script.yml

@@ -0,0 +1,135 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for eessi_container.sh script
+on: [push, pull_request, workflow_dispatch]
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  eessi_container_script:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        SCRIPT_TEST:
+        - help
+        - listrepos_default
+        - listrepos_custom
+        - run
+        - shell
+        - container
+        - resume
+        # FIXME disabled because '--access rw' is not working in CI environment
+        #- readwrite
+        #- save
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+        - name: install Apptainer
+          run: |
+            ./install_apptainer_ubuntu.sh
+
+        - name: Collect info on test environment
+          run: |
+              mount
+              df -h
+
+        - name: Test eessi_container.sh script
+          run: |
+              test_cmd="cat /etc/os-release"
+              out_pattern="Debian GNU/Linux 11"
+
+              if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
+                ./eessi_container.sh --help
+
+              # test use of --list-repos without custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
+                outfile=out_listrepos.txt
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+              # test use of --list-repos with custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
+                outfile=out_listrepos.txt
+                outfile2=out_listrepos_2.txt
+                mkdir -p ${PWD}/cfg
+                echo "[EESSI/20AB.CD]" > cfg/repos.cfg
+                echo "repo_version = 20AB.CD" >> cfg/repos.cfg
+                echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
+                echo "repo_version = 20HT.TP" >> cfg/repos.cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+                export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
+                grep "[EESSI/2023.02]" ${outfile2}
+
+              # test use of --mode run
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
+                outfile=out_run.txt
+                echo "${test_cmd}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --verbose --mode run /test/test_script.sh | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --mode shell
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then
+                outfile=out_shell.txt
+                ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --container option, using a totally different container;
+              # cfr. https://github.com/easybuilders/easybuild-containers
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then
+                outfile=out_container.txt
+                container="docker://ghcr.io/eessi/build-node:debian10"
+                ./eessi_container.sh --verbose --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "Debian GNU/Linux 10" ${outfile}
+
+              # test use of '--access rw' to get write access in container
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
+                outfile=out_readwrite.txt
+                fn="test_${RANDOM}.txt"
+                echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}
+
+                tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                # note: must use '--access rw' again here, since touched file is in overlay upper dir
+                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+
+              # test use of --resume
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
+                outfile=out_resume.txt
+                ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" > ${outfile}
+
+                tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                rm -f ${outfile}
+
+                # make sure that container image exists
+                test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1)
+
+                ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                cat ${outfile}
+                grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --save (+ --resume)
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
+                outfile=out_save.txt
+                fn="test_${RANDOM}.txt"
+                test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
+                ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+
+                tar tfv test-save.tar | grep "overlay-upper/${fn}"
+
+              else
+                  echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2
+                  exit 1
+              fi

.github/workflows/tests_scripts.yml

@@ -6,8 +6,9 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
-      - utils.sh
+      - scripts/utils.sh
       - update_lmod_cache.sh
+      - create_directory_tarballs.sh
 
   pull_request:
     branches:
@@ -16,8 +17,9 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
-      - utils.sh
+      - scripts/utils.sh
       - update_lmod_cache.sh
+      - create_directory_tarballs.sh
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
@@ -27,18 +29,9 @@ jobs:
     - name: checkout
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-    # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
     - name: install Apptainer
       run: |
-        sudo apt-get install alien
-        alien --version
-        apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
-        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/$apptainer_rpm
-        sudo alien -d $apptainer_rpm
-        sudo apt install ./apptainer*.deb
-        apptainer --version
-        # also check whether 'singularity' command is still provided by Apptainer installation
-        singularity --version
+        ./install_apptainer_ubuntu.sh
 
     - name: test install_software_layer.sh script
       run: |
@@ -49,3 +42,13 @@ jobs:
           # force using x86_64/generic, to avoid triggering an installation from scratch
           sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh
           ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh
+
+    - name: test create_directory_tarballs.sh script
+      run: |
+          # scripts need to be copied to /tmp,
+          # since create_directory_tarballs.sh must be accessible from within build container
+          cp -a * /tmp/
+          cd /tmp
+          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12
+          # check if tarballs have been produced
+          ls -l *.tar.gz

EESSI-pilot-install-software.sh

@@ -50,7 +50,7 @@ set -- "${POSITIONAL_ARGS[@]}"
 
 TOPDIR=$(dirname $(realpath $0))
 
-source $TOPDIR/utils.sh
+source $TOPDIR/scripts/utils.sh
 
 # honor $TMPDIR if it is already defined, use /tmp otherwise
 if [ -z $TMPDIR ]; then
@@ -484,6 +484,7 @@ if [ ! -f $LMOD_RC ]; then
     check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC"
 fi
 
+echo "Updating Lmod cache"
 $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
 
 $TOPDIR/check_missing_installations.sh