From 9df2fbd62b8a08147f190cc16185a781cc277885 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 14 Jun 2025 15:49:55 +0200 Subject: [PATCH 01/15] add support to and for using unionfs overlay tool --- eessi_container.sh | 60 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 906eb2cb..bdca1ec4 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -47,6 +47,7 @@ HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12)) +OVERLAY_TOOL_EXITCODE=$((${ANY_ERROR_EXITCODE} << 13)) # CernVM-FS settings CVMFS_VAR_LIB="var-lib-cvmfs" @@ -89,6 +90,9 @@ display_help() { echo " -n | --nvidia MODE - configure the container to work with NVIDIA GPUs," echo " MODE==install for a CUDA installation, MODE==run to" echo " attach a GPU, MODE==all for both [default: false]" + echo " -o | --overlay-tool ARG - tool to use to create (read-only or writable) overlay;" + echo " selected tool *must* be available in container image being used;" + echo " can be 'fuse-overlayfs' or 'unionfs' [default: fuse-overlayfs]" echo " -p | --pass-through ARG - argument to pass through to the launch of the" echo " container; can be given multiple times [default: not set]" echo " -r | --repository CFG - configuration file or identifier defining the" @@ -128,6 +132,7 @@ VERBOSE=0 STORAGE= LIST_REPOS=0 MODE="shell" +OVERLAY_TOOL="fuse-overlayfs" PASS_THROUGH=() SETUP_NVIDIA=0 REPOSITORIES=() @@ -185,6 +190,10 @@ while [[ $# -gt 0 ]]; do NVIDIA_MODE="$2" shift 2 ;; + -o|--overlay-tool) + OVERLAY_TOOL="$2" + shift 2 + ;; -p|--pass-through) PASS_THROUGH+=("$2") shift 2 @@ -779,7 +788,7 @@ do # below); the overlay-upper directory can only exist because it is part of # the ${RESUME} directory or tarball # to be able to see the contents of the read-write session we have to mount - # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository + # the overlay (in read-only mode) on top of the CernVM-FS repository echo "While processing '${cvmfs_repo_name}' to be mounted 'read-only' we detected an overlay-upper" echo " directory (${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper) likely from a previous" @@ -791,14 +800,25 @@ do EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") # now, put the overlay-upper read-only on top of the repo and make it available under the usual prefix /cvmfs - EESSI_READONLY_OVERLAY="container:fuse-overlayfs" - # The contents of the previous session are available under - # ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}. - # Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper - # the left-most directory given for the lowerdir argument is put on top, - # and with no upperdir=... the whole overlayfs is made available read-only - EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}" - EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + if [[ "${OVERLAY_TOOL}" == "fuse-overlayfs" ]]; then + EESSI_READONLY_OVERLAY="container:fuse-overlayfs" + # The contents of the previous session are available under + # ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}. + # Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper + # the left-most directory given for the lowerdir argument is put on top, + # and with no upperdir=... the whole overlayfs is made available read-only + EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}" + EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + elif [[ "${OVERLAY_TOOL}" == "unionfs" ]]; then + EESSI_READONLY_OVERLAY="container:unionfs" + # cow stands for 'copy-on-write' + EESSI_READONLY_OVERLAY+=" -o cow" + EESSI_READONLY_OVERLAY+=" /cvmfs_ro/software.eessi.io=RO" + EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + else + echo -e "ERROR: unknown overlay tool specified: ${OVERLAY_TOOL}" + exit ${OVERLAY_TOOL_EXITCODE} + fi export EESSI_READONLY_OVERLAY EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY_OVERLAY}") @@ -824,11 +844,23 @@ do EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" - EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" - EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" - EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" - EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + if [[ "${OVERLAY_TOOL}" == "fuse-overlayfs" ]]; then + EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + elif [[ "${OVERLAY_TOOL}" == "unionfs" ]]; then + # files touched are reflected under /cvmfs//.unionfs/ + EESSI_WRITABLE_OVERLAY="container:unionfs" + # cow stands for 'copy-on-write' + EESSI_WRITABLE_OVERLAY+=" -o cow" + EESSI_WRITABLE_OVERLAY+=" ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper=RW:/cvmfs_ro/software.eessi.io=RO" + EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + else + echo -e "ERROR: unknown overlay tool specified: ${OVERLAY_TOOL}" + exit ${OVERLAY_TOOL_EXITCODE} + fi export EESSI_WRITABLE_OVERLAY EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") From 955d0e604685d2c779e8c869a19a6163d13a33d6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Jun 2025 11:15:07 +0200 Subject: [PATCH 02/15] don't hardcode software.eessi.io when creating overlay in eessi_container.sh script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Röblitz --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index bdca1ec4..06ac163c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -813,7 +813,7 @@ do EESSI_READONLY_OVERLAY="container:unionfs" # cow stands for 'copy-on-write' EESSI_READONLY_OVERLAY+=" -o cow" - EESSI_READONLY_OVERLAY+=" /cvmfs_ro/software.eessi.io=RO" + EESSI_READONLY_OVERLAY+=" /cvmfs_ro/${cvmfs_repo_name}=RO" EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" else echo -e "ERROR: unknown overlay tool specified: ${OVERLAY_TOOL}" @@ -855,7 +855,7 @@ do EESSI_WRITABLE_OVERLAY="container:unionfs" # cow stands for 'copy-on-write' EESSI_WRITABLE_OVERLAY+=" -o cow" - EESSI_WRITABLE_OVERLAY+=" ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper=RW:/cvmfs_ro/software.eessi.io=RO" + EESSI_WRITABLE_OVERLAY+=" ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper=RW:/cvmfs_ro/${cvmfs_repo_name}=RO" EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" else echo -e "ERROR: unknown overlay tool specified: ${OVERLAY_TOOL}" From 1351105abf3f1fa7204ff38afdfb375cfe882fc9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Jun 2025 11:30:34 +0200 Subject: [PATCH 03/15] add test for using '--overlay-tool unionfs' option for eessi_container.sh --- .github/workflows/test_eessi_container_script.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 6f7055a5..45ca5435 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -21,6 +21,7 @@ jobs: - shell - container - resume + - unionfs # FIXME disabled because '--access rw' is not working in CI environment #- readwrite #- save @@ -135,6 +136,13 @@ jobs: tar tfv test-save.tar | grep "overlay-upper/${fn}" + # test use of --overlay-tool unionfs + elif [[ ${{matrix.SCRIPT_TEST}} == 'unionfs' ]]; then + container="docker://ghcr.io/eessi/build-node:debian12" + ./eessi_container.sh --verbose --container ${container} --access rw --overlay-tool unionfs --mode run ls -ld /cvmfs*/software.eessi.io/* 2>&1 | tee ${outfile} + grep "/cvmfs/software.eessi.io/versions" $outfile + grep "/cvmfs_ro/software.eessi.io/versions" $outfile + else echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2 exit 1 From dd4cc2ac0128ad28d71ba6f9bda8445c3645b2c0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Jun 2025 11:30:34 +0200 Subject: [PATCH 04/15] add test for using '--overlay-tool unionfs' option for eessi_container.sh --- .github/workflows/test_eessi_container_script.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 6f7055a5..be0341c8 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -21,6 +21,7 @@ jobs: - shell - container - resume + - unionfs # FIXME disabled because '--access rw' is not working in CI environment #- readwrite #- save @@ -135,6 +136,15 @@ jobs: tar tfv test-save.tar | grep "overlay-upper/${fn}" + # test use of --overlay-tool unionfs + elif [[ ${{matrix.SCRIPT_TEST}} == 'unionfs' ]]; then + container="docker://ghcr.io/eessi/build-node:debian12" + echo 'ls -ld /cvmfs*/software.eessi.io/*' > test_script.sh + chmod u+x test_script.sh + ./eessi_container.sh --verbose --container ${container} --access rw --overlay-tool unionfs --mode run /test/test_script.sh 2>&1 | tee ${outfile} + grep "/cvmfs/software.eessi.io/versions" $outfile + grep "/cvmfs_ro/software.eessi.io/versions" $outfile + else echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2 exit 1 From b421f1d1b4a99f5063e7ed02d19a4ee287b1fc8f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Jun 2025 11:56:50 +0200 Subject: [PATCH 05/15] replace EESSI version placeholder in CI workflow for scripts --- .github/workflows/tests_scripts.yml | 44 ++++++++++++++++------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index de227dfc..29fd2146 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -29,6 +29,11 @@ permissions: jobs: build: runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - '2023.06' steps: - name: checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -39,8 +44,8 @@ jobs: - name: test load_easybuild_module.sh script run: | - # bind current directory into container as /software-layer - export SINGULARITY_BIND="${PWD}:/software-layer" + # bind current directory into container as /software-layer-scripts + export SINGULARITY_BIND="${PWD}:/software-layer-scripts" # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06, # since Python in compat layer is Python 3.11.x; @@ -58,15 +63,16 @@ jobs: echo 'export TMPDIR=$(mktemp -d)' >> ${test_script} # set up environment to have utility functions in place that load_easybuild_module.sh script relies on, # along with $EESSI_* environment variables, and Lmod - echo 'ls -l /software-layer/' >> ${test_script} - echo 'source /software-layer/scripts/utils.sh' >> ${test_script} - echo 'source /software-layer/init/eessi_environment_variables' >> ${test_script} + echo 'ls -l /software-layer-scripts/' >> ${test_script} + echo 'sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/eessi_defaults + echo 'source /software-layer-scripts/scripts/utils.sh' >> ${test_script} + echo 'source /software-layer-scripts/init/eessi_environment_variables' >> ${test_script} echo 'source ${EPREFIX}/usr/share/Lmod/init/bash' >> ${test_script} # minimal configuration for EasyBuild so we can test installation aspect of load_easybuild_module.sh script echo "export EASYBUILD_INSTALLPATH=/tmp/eb-${EB_VERSION}" >> ${test_script} echo 'module use ${EASYBUILD_INSTALLPATH}/modules/all' >> ${test_script} echo '' >> ${test_script} - echo "source /software-layer/load_easybuild_module.sh ${EB_VERSION}" >> ${test_script} + echo "source /software-layer-scripts/load_easybuild_module.sh ${EB_VERSION}" >> ${test_script} echo 'module list' >> ${test_script} echo 'eb --version' >> ${test_script} @@ -74,48 +80,48 @@ jobs: # run wrapper script + capture & check output out="${PWD}/eb-${EB_VERSION}.out" - ./eessi_container.sh --access rw --mode run --verbose /software-layer/run_in_compat_layer_env.sh /software-layer/eb-${EB_VERSION}.sh 2>&1 | tee ${out} + ./eessi_container.sh --access rw --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/eb-${EB_VERSION}.sh 2>&1 | tee ${out} pattern="^This is EasyBuild ${EB_VERSION} " grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1) done - name: test install_software_layer.sh script run: | - # bind current directory into container as /software-layer - export SINGULARITY_BIND="${PWD}:/software-layer" + # bind current directory into container as /software-layer-scripts + export SINGULARITY_BIND="${PWD}:/software-layer-scripts" # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh # skip installation of CUDA SDKs, since this is too heavy for CI sed -i "s@./EESSI-install-software.sh@./EESSI-install-software.sh --skip-cuda-install@g" install_software_layer.sh - ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh + ./eessi_container.sh --mode run --verbose /software-layer-scripts/install_software_layer.sh - name: test create_directory_tarballs.sh script run: | - # bind current directory into container as /software-layer - export SINGULARITY_BIND="${PWD}:/software-layer" + # bind current directory into container as /software-layer-scripts + export SINGULARITY_BIND="${PWD}:/software-layer-scripts" # scripts need to be copied to /tmp, # since create_directory_tarballs.sh must be accessible from within build container - ./eessi_container.sh --mode run --verbose /software-layer/create_directory_tarballs.sh 2023.06 + ./eessi_container.sh --mode run --verbose /software-layer-scripts/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz - name: test create_lmodsitepackage.py script run: | - # bind current directory into container as /software-layer - export SINGULARITY_BIND="${PWD}:/software-layer" + # bind current directory into container as /software-layer-scripts + export SINGULARITY_BIND="${PWD}:/software-layer-scripts" - # Creates .lmod/SitePackage.lua in current dir, which then gets bind-mounted into /software-layer + # Creates .lmod/SitePackage.lua in current dir, which then gets bind-mounted into /software-layer-scripts python3 create_lmodsitepackage.py . # run some commands to make sure that generated Lmod SitePackage file works test_script="${PWD}/test_lmod_sitepackage.sh" echo '#!/bin/bash' > ${test_script} - echo 'export LMOD_PACKAGE_PATH="/software-layer/.lmod"' > ${test_script} + echo 'export LMOD_PACKAGE_PATH="/software-layer-scripts/.lmod"' > ${test_script} echo 'ml --config' >> ${test_script} chmod u+x ${test_script} out="${PWD}/test_create_lmodsitepackage.out" - ./eessi_container.sh --mode run --verbose /software-layer/run_in_compat_layer_env.sh /software-layer/test_lmod_sitepackage.sh 2>&1 | tee ${out} - for pattern in "^Site Pkg location.*/software-layer/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer/.lmod/SitePackage.lua"; do + ./eessi_container.sh --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/test_lmod_sitepackage.sh 2>&1 | tee ${out} + for pattern in "^Site Pkg location.*/software-layer-scripts/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer-scripts/.lmod/SitePackage.lua"; do grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1) done From d96fb6f6e0b23e071837146850d5452056714f2b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Jun 2025 12:04:10 +0200 Subject: [PATCH 06/15] fix test of using unionfs as overlay-tool --- .github/workflows/test_eessi_container_script.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index be0341c8..4a05c3ec 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -139,6 +139,7 @@ jobs: # test use of --overlay-tool unionfs elif [[ ${{matrix.SCRIPT_TEST}} == 'unionfs' ]]; then container="docker://ghcr.io/eessi/build-node:debian12" + export SINGULARITY_BIND="$PWD:/test" echo 'ls -ld /cvmfs*/software.eessi.io/*' > test_script.sh chmod u+x test_script.sh ./eessi_container.sh --verbose --container ${container} --access rw --overlay-tool unionfs --mode run /test/test_script.sh 2>&1 | tee ${outfile} From c9b81ba8d4b73a75798529012c866ab942a6da2f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Jun 2025 15:30:51 +0200 Subject: [PATCH 07/15] improve pattern check in test for using unionfs as overlay-tool --- .github/workflows/test_eessi_container_script.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 4a05c3ec..76948210 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -143,8 +143,9 @@ jobs: echo 'ls -ld /cvmfs*/software.eessi.io/*' > test_script.sh chmod u+x test_script.sh ./eessi_container.sh --verbose --container ${container} --access rw --overlay-tool unionfs --mode run /test/test_script.sh 2>&1 | tee ${outfile} - grep "/cvmfs/software.eessi.io/versions" $outfile - grep "/cvmfs_ro/software.eessi.io/versions" $outfile + for pattern in "/cvmfs/software.eessi.io/versions" "/cvmfs_ro/software.eessi.io/versions"; do + grep "${pattern}" $outfile || (echo "Pattern '${pattern}' not found in $outfile"; exit 1) + done else echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2 From 2bbadca455600e131bbfbb8975ee2b5eee844d53 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Jun 2025 21:01:05 +0200 Subject: [PATCH 08/15] fix checking for use of '--overlay-tool unionfs' --- .github/workflows/test_eessi_container_script.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 76948210..c281fe07 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -138,13 +138,14 @@ jobs: # test use of --overlay-tool unionfs elif [[ ${{matrix.SCRIPT_TEST}} == 'unionfs' ]]; then + outfile=out_unionfs.txt container="docker://ghcr.io/eessi/build-node:debian12" export SINGULARITY_BIND="$PWD:/test" echo 'ls -ld /cvmfs*/software.eessi.io/*' > test_script.sh chmod u+x test_script.sh ./eessi_container.sh --verbose --container ${container} --access rw --overlay-tool unionfs --mode run /test/test_script.sh 2>&1 | tee ${outfile} for pattern in "/cvmfs/software.eessi.io/versions" "/cvmfs_ro/software.eessi.io/versions"; do - grep "${pattern}" $outfile || (echo "Pattern '${pattern}' not found in $outfile"; exit 1) + grep "${pattern}" ${outfile} || (echo "Pattern '${pattern}' not found in ${outfile}"; exit 1) done else From 8e2e292ca52a0cc72cc3dd313167a357bc4a76bc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Jun 2025 21:01:53 +0200 Subject: [PATCH 09/15] fix replacing of EESSI version placeholder in CI workflow for scripts --- .github/workflows/tests_scripts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 29fd2146..bb94db21 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -64,7 +64,7 @@ jobs: # set up environment to have utility functions in place that load_easybuild_module.sh script relies on, # along with $EESSI_* environment variables, and Lmod echo 'ls -l /software-layer-scripts/' >> ${test_script} - echo 'sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/eessi_defaults + echo 'sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/eessi_defaults' >> ${test_script} echo 'source /software-layer-scripts/scripts/utils.sh' >> ${test_script} echo 'source /software-layer-scripts/init/eessi_environment_variables' >> ${test_script} echo 'source ${EPREFIX}/usr/share/Lmod/init/bash' >> ${test_script} From 1675ff93e74e70dc01ded947bbc6a3187d94895b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Jun 2025 15:19:28 +0200 Subject: [PATCH 10/15] set $EESSI_VERSION_OVERRIDE rather than replacing __EESSI_VERSION_DEFAULT__ placeholder in init/eessi_defaults in CI workflow to test load_easybuild_module.sh script --- .github/workflows/tests_scripts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index bb94db21..3acadb78 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -64,7 +64,7 @@ jobs: # set up environment to have utility functions in place that load_easybuild_module.sh script relies on, # along with $EESSI_* environment variables, and Lmod echo 'ls -l /software-layer-scripts/' >> ${test_script} - echo 'sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/eessi_defaults' >> ${test_script} + echo 'export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"' >> ${test_script} echo 'source /software-layer-scripts/scripts/utils.sh' >> ${test_script} echo 'source /software-layer-scripts/init/eessi_environment_variables' >> ${test_script} echo 'source ${EPREFIX}/usr/share/Lmod/init/bash' >> ${test_script} From 00f3ea28788886cf632981531f8f4b97a4da6809 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 24 Jun 2025 11:16:25 +0200 Subject: [PATCH 11/15] show contents of test script when testing load_easybuild_module.sh --- .github/workflows/tests_scripts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 3acadb78..6b30fa71 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -77,6 +77,8 @@ jobs: echo 'eb --version' >> ${test_script} chmod u+x ${test_script} + echo ">> Contents of ${test_script}:" + cat ${test_script} # run wrapper script + capture & check output out="${PWD}/eb-${EB_VERSION}.out" From f9392e6fbcccb232ea2a8fb07e8843916138ce39 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Jun 2025 10:15:02 +0200 Subject: [PATCH 12/15] set $EESSI_VERSION_OVERRIDE before calling run_in_compat_layer_env.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .github/workflows/tests_scripts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 6b30fa71..19c1c437 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -82,6 +82,7 @@ jobs: # run wrapper script + capture & check output out="${PWD}/eb-${EB_VERSION}.out" + export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}" ./eessi_container.sh --access rw --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/eb-${EB_VERSION}.sh 2>&1 | tee ${out} pattern="^This is EasyBuild ${EB_VERSION} " grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1) From dd37dfdd2370214b90d488407fe0c140b9a04458 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Jun 2025 10:36:12 +0200 Subject: [PATCH 13/15] also set $EESSI_VERSION_OVERRIDE for install_software_layer.sh script tests --- .github/workflows/tests_scripts.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 19c1c437..0289b871 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -77,12 +77,12 @@ jobs: echo 'eb --version' >> ${test_script} chmod u+x ${test_script} - echo ">> Contents of ${test_script}:" - cat ${test_script} + + # make sure that correct EESSI version is used (required because default is a placeholder version) + export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}" # run wrapper script + capture & check output out="${PWD}/eb-${EB_VERSION}.out" - export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}" ./eessi_container.sh --access rw --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/eb-${EB_VERSION}.sh 2>&1 | tee ${out} pattern="^This is EasyBuild ${EB_VERSION} " grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1) @@ -96,12 +96,17 @@ jobs: sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh # skip installation of CUDA SDKs, since this is too heavy for CI sed -i "s@./EESSI-install-software.sh@./EESSI-install-software.sh --skip-cuda-install@g" install_software_layer.sh + + # make sure that correct EESSI version is used (required because default is a placeholder version) + export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}" + ./eessi_container.sh --mode run --verbose /software-layer-scripts/install_software_layer.sh - name: test create_directory_tarballs.sh script run: | # bind current directory into container as /software-layer-scripts export SINGULARITY_BIND="${PWD}:/software-layer-scripts" + # scripts need to be copied to /tmp, # since create_directory_tarballs.sh must be accessible from within build container ./eessi_container.sh --mode run --verbose /software-layer-scripts/create_directory_tarballs.sh 2023.06 From 5f4b5573f3bfae9c5f4a879e74ecce6fe12de178 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Jun 2025 12:06:13 +0200 Subject: [PATCH 14/15] also set $EESSI_VERSION_OVERRIDE for create_directory_tarballs.sh script tests --- .github/workflows/tests_scripts.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 0289b871..9ffc3f2c 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -107,6 +107,9 @@ jobs: # bind current directory into container as /software-layer-scripts export SINGULARITY_BIND="${PWD}:/software-layer-scripts" + # make sure that correct EESSI version is used (required because default is a placeholder version) + export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}" + # scripts need to be copied to /tmp, # since create_directory_tarballs.sh must be accessible from within build container ./eessi_container.sh --mode run --verbose /software-layer-scripts/create_directory_tarballs.sh 2023.06 From 0ec75f10673843121cf1110e6114896947270fc7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 27 Jun 2025 15:49:09 +0200 Subject: [PATCH 15/15] also set for create_lmodsitepackage.py tests --- .github/workflows/tests_scripts.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 9ffc3f2c..9fd6421c 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -131,6 +131,9 @@ jobs: chmod u+x ${test_script} + # make sure that correct EESSI version is used (required because default is a placeholder version) + export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}" + out="${PWD}/test_create_lmodsitepackage.out" ./eessi_container.sh --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh /software-layer-scripts/test_lmod_sitepackage.sh 2>&1 | tee ${out} for pattern in "^Site Pkg location.*/software-layer-scripts/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer-scripts/.lmod/SitePackage.lua"; do