add support to eessi_container.sh for using unionfs overlay tool

.github/workflows/test_eessi_container_script.yml

@@ -21,6 +21,7 @@ jobs:
         - shell
         - container
         - resume
+        - unionfs
         # FIXME disabled because '--access rw' is not working in CI environment
         #- readwrite
         #- save
@@ -135,6 +136,18 @@ jobs:
 
                 tar tfv test-save.tar | grep "overlay-upper/${fn}"
 
+              # test use of --overlay-tool unionfs
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'unionfs' ]]; then
+                outfile=out_unionfs.txt
+                container="docker://ghcr.io/eessi/build-node:debian12"
+                export SINGULARITY_BIND="$PWD:/test"
+                echo 'ls -ld /cvmfs*/software.eessi.io/*' > test_script.sh
+                chmod u+x test_script.sh
+                ./eessi_container.sh --verbose --container ${container} --access rw --overlay-tool unionfs --mode run /test/test_script.sh 2>&1 | tee ${outfile}
+                for pattern in "/cvmfs/software.eessi.io/versions" "/cvmfs_ro/software.eessi.io/versions"; do
+                  grep "${pattern}" ${outfile} || (echo "Pattern '${pattern}' not found in ${outfile}"; exit 1)
+                done
+
               else
                   echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2
                   exit 1

.github/workflows/tests_scripts.yml

@@ -29,6 +29,11 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+          - '2023.06'
     steps:
     - name: checkout
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -39,8 +44,8 @@ jobs:
 
     - name: test load_easybuild_module.sh script
       run: |
-        # bind current directory into container as /software-layer
-        export SINGULARITY_BIND="${PWD}:/software-layer"
+        # bind current directory into container as /software-layer-scripts
+        export SINGULARITY_BIND="${PWD}:/software-layer-scripts"
 
         # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06,
         # since Python in compat layer is Python 3.11.x;
@@ -58,64 +63,79 @@ jobs:
           echo 'export TMPDIR=$(mktemp -d)' >> ${test_script}
           # set up environment to have utility functions in place that load_easybuild_module.sh script relies on,
           # along with $EESSI_* environment variables, and Lmod
-          echo 'ls -l /software-layer/' >> ${test_script}
-          echo 'source /software-layer/scripts/utils.sh' >> ${test_script}
-          echo 'source /software-layer/init/eessi_environment_variables' >> ${test_script}
+          echo 'ls -l /software-layer-scripts/' >> ${test_script}
+          echo 'export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"' >> ${test_script}
+          echo 'source /software-layer-scripts/scripts/utils.sh' >> ${test_script}
+          echo 'source /software-layer-scripts/init/eessi_environment_variables' >> ${test_script}
           echo 'source ${EPREFIX}/usr/share/Lmod/init/bash' >> ${test_script}
           # minimal configuration for EasyBuild so we can test installation aspect of load_easybuild_module.sh script
           echo "export EASYBUILD_INSTALLPATH=/tmp/eb-${EB_VERSION}" >> ${test_script}
           echo 'module use ${EASYBUILD_INSTALLPATH}/modules/all' >> ${test_script}
           echo '' >> ${test_script}
-          echo "source /software-layer/load_easybuild_module.sh ${EB_VERSION}" >> ${test_script}
+          echo "source /software-layer-scripts/load_easybuild_module.sh ${EB_VERSION}" >> ${test_script}
           echo 'module list' >> ${test_script}
           echo 'eb --version' >> ${test_script}
 
           chmod u+x ${test_script}
 
+          # make sure that correct EESSI version is used (required because default is a placeholder version)
+          export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"
+
           # run wrapper script + capture & check output
           out="${PWD}/eb-${EB_VERSION}.out"
-          ./eessi_container.sh --access rw --mode run --verbose /software-layer/run_in_compat_layer_env.sh /software-layer/eb-${EB_VERSION}.sh 2>&1 | tee ${out}
+          ./eessi_container.sh --access rw --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/eb-${EB_VERSION}.sh 2>&1 | tee ${out}
           pattern="^This is EasyBuild ${EB_VERSION} "
           grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1)
         done
 
     - name: test install_software_layer.sh script
       run: |
-          # bind current directory into container as /software-layer
-          export SINGULARITY_BIND="${PWD}:/software-layer"
+          # bind current directory into container as /software-layer-scripts
+          export SINGULARITY_BIND="${PWD}:/software-layer-scripts"
           # force using x86_64/generic, to avoid triggering an installation from scratch
           sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh
           # skip installation of CUDA SDKs, since this is too heavy for CI
           sed -i "s@./EESSI-install-software.sh@./EESSI-install-software.sh --skip-cuda-install@g" install_software_layer.sh
-          ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh
+
+          # make sure that correct EESSI version is used (required because default is a placeholder version)
+          export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"
+
+          ./eessi_container.sh --mode run --verbose /software-layer-scripts/install_software_layer.sh
 
     - name: test create_directory_tarballs.sh script
       run: |
-          # bind current directory into container as /software-layer
-          export SINGULARITY_BIND="${PWD}:/software-layer"
+          # bind current directory into container as /software-layer-scripts
+          export SINGULARITY_BIND="${PWD}:/software-layer-scripts"
+
+          # make sure that correct EESSI version is used (required because default is a placeholder version)
+          export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"
+
           # scripts need to be copied to /tmp,
           # since create_directory_tarballs.sh must be accessible from within build container
-          ./eessi_container.sh --mode run --verbose /software-layer/create_directory_tarballs.sh 2023.06
+          ./eessi_container.sh --mode run --verbose /software-layer-scripts/create_directory_tarballs.sh 2023.06
           # check if tarballs have been produced
           ls -l *.tar.gz
 
     - name: test create_lmodsitepackage.py script
       run: |
-          # bind current directory into container as /software-layer
-          export SINGULARITY_BIND="${PWD}:/software-layer"
+          # bind current directory into container as /software-layer-scripts
+          export SINGULARITY_BIND="${PWD}:/software-layer-scripts"
 
-          # Creates .lmod/SitePackage.lua in current dir, which then gets bind-mounted into /software-layer
+          # Creates .lmod/SitePackage.lua in current dir, which then gets bind-mounted into /software-layer-scripts
           python3 create_lmodsitepackage.py .
           # run some commands to make sure that generated Lmod SitePackage file works
           test_script="${PWD}/test_lmod_sitepackage.sh"
           echo '#!/bin/bash' > ${test_script}
-          echo 'export LMOD_PACKAGE_PATH="/software-layer/.lmod"' > ${test_script}
+          echo 'export LMOD_PACKAGE_PATH="/software-layer-scripts/.lmod"' > ${test_script}
           echo 'ml --config' >> ${test_script}
 
           chmod u+x ${test_script}
 
+          # make sure that correct EESSI version is used (required because default is a placeholder version)
+          export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"
+
           out="${PWD}/test_create_lmodsitepackage.out"
-          ./eessi_container.sh --mode run --verbose /software-layer/run_in_compat_layer_env.sh /software-layer/test_lmod_sitepackage.sh 2>&1 | tee ${out}
-          for pattern in "^Site Pkg location.*/software-layer/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer/.lmod/SitePackage.lua"; do
+          ./eessi_container.sh --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/test_lmod_sitepackage.sh 2>&1 | tee ${out}
+          for pattern in "^Site Pkg location.*/software-layer-scripts/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer-scripts/.lmod/SitePackage.lua"; do
             grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1)
           done

eessi_container.sh

@@ -47,6 +47,7 @@ HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
 HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12))
+OVERLAY_TOOL_EXITCODE=$((${ANY_ERROR_EXITCODE} << 13))
 
 # CernVM-FS settings
 CVMFS_VAR_LIB="var-lib-cvmfs"
@@ -89,6 +90,9 @@ display_help() {
   echo "  -n | --nvidia MODE      - configure the container to work with NVIDIA GPUs,"
   echo "                            MODE==install for a CUDA installation, MODE==run to"
   echo "                            attach a GPU, MODE==all for both [default: false]"
+  echo "  -o | --overlay-tool ARG - tool to use to create (read-only or writable) overlay;"
+  echo "                            selected tool *must* be available in container image being used;"
+  echo "                            can be 'fuse-overlayfs' or 'unionfs' [default: fuse-overlayfs]"
   echo "  -p | --pass-through ARG - argument to pass through to the launch of the"
   echo "                            container; can be given multiple times [default: not set]"
   echo "  -r | --repository CFG   - configuration file or identifier defining the"
@@ -128,6 +132,7 @@ VERBOSE=0
 STORAGE=
 LIST_REPOS=0
 MODE="shell"
+OVERLAY_TOOL="fuse-overlayfs"
 PASS_THROUGH=()
 SETUP_NVIDIA=0
 REPOSITORIES=()
@@ -185,6 +190,10 @@ while [[ $# -gt 0 ]]; do
       NVIDIA_MODE="$2"
       shift 2
       ;;
+    -o|--overlay-tool)
+      OVERLAY_TOOL="$2"
+      shift 2
+      ;;
     -p|--pass-through)
       PASS_THROUGH+=("$2")
       shift 2
@@ -779,7 +788,7 @@ do
             # below); the overlay-upper directory can only exist because it is part of
             # the ${RESUME} directory or tarball
             # to be able to see the contents of the read-write session we have to mount
-            # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository
+            # the overlay (in read-only mode) on top of the CernVM-FS repository
 
             echo "While processing '${cvmfs_repo_name}' to be mounted 'read-only' we detected an overlay-upper"
             echo "  directory (${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper) likely from a previous"
@@ -791,14 +800,25 @@ do
             EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
 
             # now, put the overlay-upper read-only on top of the repo and make it available under the usual prefix /cvmfs
-            EESSI_READONLY_OVERLAY="container:fuse-overlayfs"
-            # The contents of the previous session are available under
-            #   ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}.
-            #   Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper
-            # the left-most directory given for the lowerdir argument is put on top,
-            #   and with no upperdir=... the whole overlayfs is made available read-only
-            EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}"
-            EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+            if [[ "${OVERLAY_TOOL}" == "fuse-overlayfs" ]]; then
+                EESSI_READONLY_OVERLAY="container:fuse-overlayfs"
+                # The contents of the previous session are available under
+                #   ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}.
+                #   Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper
+                # the left-most directory given for the lowerdir argument is put on top,
+                #   and with no upperdir=... the whole overlayfs is made available read-only
+                EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}"
+                EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+            elif [[ "${OVERLAY_TOOL}" == "unionfs" ]]; then
+                EESSI_READONLY_OVERLAY="container:unionfs"
+                # cow stands for 'copy-on-write'
+                EESSI_READONLY_OVERLAY+=" -o cow"
+                EESSI_READONLY_OVERLAY+=" /cvmfs_ro/${cvmfs_repo_name}=RO"
+                EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+            else
+                echo -e "ERROR: unknown overlay tool specified: ${OVERLAY_TOOL}"
+                exit ${OVERLAY_TOOL_EXITCODE}
+            fi
             export EESSI_READONLY_OVERLAY
 
             EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY_OVERLAY}")
@@ -824,11 +844,23 @@ do
 
         EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
 
-        EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"
-        EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}"
-        EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper"
-        EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work"
-        EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+        if [[ "${OVERLAY_TOOL}" == "fuse-overlayfs" ]]; then
+            EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"
+            EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}"
+            EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper"
+            EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work"
+            EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+        elif [[ "${OVERLAY_TOOL}" == "unionfs" ]]; then
+            # files touched are reflected under /cvmfs/<repo>/.unionfs/
+            EESSI_WRITABLE_OVERLAY="container:unionfs"
+            # cow stands for 'copy-on-write'
+            EESSI_WRITABLE_OVERLAY+=" -o cow"
+            EESSI_WRITABLE_OVERLAY+=" ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper=RW:/cvmfs_ro/${cvmfs_repo_name}=RO"
+            EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+        else
+            echo -e "ERROR: unknown overlay tool specified: ${OVERLAY_TOOL}"
+            exit ${OVERLAY_TOOL_EXITCODE}
+        fi
         export EESSI_WRITABLE_OVERLAY
 
         EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}")