From 4e2f6d046a673f7d22106c505b116e5dc2cb7de5 Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Tue, 20 Dec 2022 10:23:17 +0100
Subject: [PATCH 01/78] Move bash utils to new scripts dir, add new dir to
 tarball creation

---
 .github/workflows/tests_scripts.yml           |  4 +--
 EESSI-pilot-install-software.sh               |  2 +-
 ...tarball.sh => create_directory_tarballs.sh | 33 ++++++++++---------
 utils.sh => scripts/utils.sh                  |  0
 update_lmod_cache.sh                          |  2 +-
 5 files changed, 22 insertions(+), 19 deletions(-)
 rename create_init_tarball.sh => create_directory_tarballs.sh (52%)
 rename utils.sh => scripts/utils.sh (100%)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 18fcd7b255..d224034c2a 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -6,7 +6,7 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
-      - utils.sh
+      - scripts/utils.sh
       - update_lmod_cache.sh
 
   pull_request:
@@ -16,7 +16,7 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
-      - utils.sh
+      - scripts/utils.sh
       - update_lmod_cache.sh
 permissions:
   contents: read # to fetch code (actions/checkout)
diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 10311b7a15..4ca553dbe7 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -50,7 +50,7 @@ set -- "${POSITIONAL_ARGS[@]}"
 
 TOPDIR=$(dirname $(realpath $0))
 
-source $TOPDIR/utils.sh
+source $TOPDIR/scripts/utils.sh
 
 # honor $TMPDIR if it is already defined, use /tmp otherwise
 if [ -z $TMPDIR ]; then
diff --git a/create_init_tarball.sh b/create_directory_tarballs.sh
similarity index 52%
rename from create_init_tarball.sh
rename to create_directory_tarballs.sh
index 3393285a33..c254c2c229 100755
--- a/create_init_tarball.sh
+++ b/create_directory_tarballs.sh
@@ -4,19 +4,6 @@ SOFTWARE_LAYER_TARBALL_URL=https://github.com/EESSI/software-layer/tarball/main
 
 set -eo pipefail
 
-function echo_green() {
-    echo -e "\e[32m$1\e[0m"
-}
-
-function echo_red() {
-    echo -e "\e[31m$1\e[0m"
-}
-
-function error() {
-    echo_red "ERROR: $1" >&2
-    exit 1
-}
-
 if [ $# -ne 1 ]; then
     echo "Usage: $0 <EESSI version>" >&2
     exit 1
@@ -24,13 +11,18 @@ fi
 
 version=$1
 
+TOPDIR=$(dirname $(realpath $0))
+
+source $TOPDIR/scripts/utils.sh
+
 # Check if the EESSI version number encoded in the filename
 # is valid, i.e. matches the format YYYY.DD
 if ! echo "${version}" | egrep -q '^20[0-9][0-9]\.(0[0-9]|1[0-2])$'
 then
-    error "${version} is not a valid EESSI version."
+    fatal_error "${version} is not a valid EESSI version."
 fi
 
+# Create tarball of init directory
 tartmp=$(mktemp -t -d init.XXXXX)
 mkdir "${tartmp}/${version}"
 tarname="eessi-${version}-init-$(date +%s).tar.gz"
@@ -38,9 +30,20 @@ curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --s
 source "${tartmp}/${version}/init/minimal_eessi_env"
 if [ "${EESSI_PILOT_VERSION}" != "${version}" ]
 then
-  error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!"
+  fatal_error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!"
 fi
 tar czf "${tarname}" -C "${tartmp}" "${version}"
 rm -rf "${tartmp}"
 
 echo_green "Done! Created tarball ${tarname}."
+
+# Create tarball of scripts directory
+# Version check has already been performed and would have caused script to exit at this point in case of problems
+tartmp=$(mktemp -t -d scripts.XXXXX)
+mkdir "${tartmp}/${version}"
+tarname="eessi-${version}-scripts-$(date +%s).tar.gz"
+curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards */scripts/
+tar czf "${tarname}" -C "${tartmp}" "${version}"
+rm -rf "${tartmp}"
+
+echo_green "Done! Created tarball ${tarname}."
diff --git a/utils.sh b/scripts/utils.sh
similarity index 100%
rename from utils.sh
rename to scripts/utils.sh
diff --git a/update_lmod_cache.sh b/update_lmod_cache.sh
index 89e2ecbeee..814bb2dae0 100755
--- a/update_lmod_cache.sh
+++ b/update_lmod_cache.sh
@@ -5,7 +5,7 @@
 
 TOPDIR=$(dirname $(realpath $0))
 
-source $TOPDIR/utils.sh
+source $TOPDIR/scripts/utils.sh
 
 if [ $# -ne 2 ]; then
     echo "Usage: $0 <path to compat layer directory> <path to software installation prefix>" >&2

From 8cac3dfeb43a59f841f33baa62330bb38fe29f01 Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Wed, 25 Jan 2023 14:37:33 +0100
Subject: [PATCH 02/78] Update of where utils.sh is located when sourcing

---
 check_missing_installations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 30f9cc6ff7..e4c75aebd8 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -16,7 +16,7 @@ fi
 
 LOCAL_TMPDIR=$(mktemp -d)
 
-source $TOPDIR/utils.sh
+source $TOPDIR/scripts/utils.sh
 
 source $TOPDIR/configure_easybuild
 

From 3390a2d56ab5c28200ff487e124c74e87945679c Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Wed, 25 Jan 2023 14:40:06 +0100
Subject: [PATCH 03/78] Add create_directory_tarballs.sh to github CI

---
 .github/workflows/tests_scripts.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index dbf8933555..53f4bd13be 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -8,6 +8,7 @@ on:
       - run_in_compat_layer_env.sh
       - scripts/utils.sh
       - update_lmod_cache.sh
+      - create_directory_tarballs.sh
 
   pull_request:
     branches:
@@ -18,6 +19,7 @@ on:
       - run_in_compat_layer_env.sh
       - scripts/utils.sh
       - update_lmod_cache.sh
+      - create_directory_tarballs.sh
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:

From f513b120b7264c0a4af1479feac533d86b95b24f Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Thu, 2 Feb 2023 15:35:42 +0100
Subject: [PATCH 04/78] Add CI test for the create_directory_tarballs.sh script

---
 .github/workflows/tests_scripts.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 53f4bd13be..a78c71dc9a 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -51,3 +51,13 @@ jobs:
           # force using x86_64/generic, to avoid triggering an installation from scratch
           sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh
           ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh
+
+    - name: test create_directory_tarballs.sh script
+      run: |
+          # scripts need to be copied to /tmp,
+          # since create_directory_tarballs.sh must be accessible from within build container
+          cp -a * /tmp/
+          cd /tmp
+          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh
+          # check if tarballs have been produced
+          ls -l *.tar.gz

From e4702ee06c71fba563e9d4f362fdf3aac41d394f Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Thu, 2 Feb 2023 17:47:24 +0100
Subject: [PATCH 05/78] Need to specify the EESSI version to test
 create_directory_tarballs

---
 .github/workflows/tests_scripts.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index a78c71dc9a..23bb36a7e8 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -58,6 +58,6 @@ jobs:
           # since create_directory_tarballs.sh must be accessible from within build container
           cp -a * /tmp/
           cd /tmp
-          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh
+          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12
           # check if tarballs have been produced
           ls -l *.tar.gz

From c4b9cf5e2a4dcd75fca54e9fc9da2c6b69011d9e Mon Sep 17 00:00:00 2001
From: Michael Huebner <huebner@physik.uni-bonn.de>
Date: Thu, 2 Feb 2023 18:15:34 +0100
Subject: [PATCH 06/78] Fix wildcard usage when building tarball of the scripts
 dir

---
 create_directory_tarballs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/create_directory_tarballs.sh b/create_directory_tarballs.sh
index c254c2c229..be282463eb 100755
--- a/create_directory_tarballs.sh
+++ b/create_directory_tarballs.sh
@@ -42,7 +42,7 @@ echo_green "Done! Created tarball ${tarname}."
 tartmp=$(mktemp -t -d scripts.XXXXX)
 mkdir "${tartmp}/${version}"
 tarname="eessi-${version}-scripts-$(date +%s).tar.gz"
-curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards */scripts/
+curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards '*/scripts/'
 tar czf "${tarname}" -C "${tartmp}" "${version}"
 rm -rf "${tartmp}"
 

From 3a6fe0e4ec0b10387bfd43cf346c3f052b4b85f3 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 08:49:37 +0100
Subject: [PATCH 07/78] update location of utils.sh, uniform way to determine
 'TOPDIR'

---
 eessi_container.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 57b003368b..29a2ea4565 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -23,13 +23,13 @@
 # 6. run container
 
 # -. initial settings & exit codes
-base_dir=$(dirname $(realpath $0))
+TOPDIR=$(dirname $(realpath $0))
 
-source ${base_dir}/utils.sh
-source ${base_dir}/cfg_files.sh
+source ${TOPDIR}/scripts/utils.sh
+source ${TOPDIR}/cfg_files.sh
 
 # exit codes: bitwise shift codes to allow for combination of exit codes
-# ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh
+# ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh
 CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1))
 ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2))
 CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3))

From 756b1b62407c250dfe8b7dcf9d87511375c7580c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 08:53:54 +0100
Subject: [PATCH 08/78] bump default container version

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 29a2ea4565..2b2c0716ca 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -61,7 +61,7 @@ display_help() {
   echo " OPTIONS:"
   echo "  -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]"
   echo "  -c | --container IMG  - image file or URL defining the container to use"
-  echo "                          [default: docker://ghcr.io/eessi/build-node:debian10]"
+  echo "                          [default: docker://ghcr.io/eessi/build-node:debian11]"
   echo "  -h | --help           - display this usage information [default: false]"
   echo "  -g | --storage DIR    - directory space on host machine (used for"
   echo "                          temporary data) [default: 1. TMPDIR, 2. /tmp]"
@@ -95,7 +95,7 @@ display_help() {
 
 # set defaults for command line arguments
 ACCESS="ro"
-CONTAINER="docker://ghcr.io/eessi/build-node:debian10"
+CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 DRY_RUN=0
 INFO=0
 STORAGE=

From 4c435ab94185c8ab1cb64eebc97ffa96d47e83fb Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 08:57:00 +0100
Subject: [PATCH 09/78] clarifying default value for repository arg

---
 eessi_container.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 2b2c0716ca..b6eb3894ef 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -68,7 +68,8 @@ display_help() {
   echo "  -m | --mode MODE      - with MODE==shell (launch interactive shell) or"
   echo "                          MODE==run (run a script) [default: shell]"
   echo "  -r | --repository CFG - configuration file or identifier defining the"
-  echo "                          repository to use [default: EESSI-pilot]"
+  echo "                          repository to use [default: EESSI-pilot via"
+  echo "                          container configuration]"
   echo "  -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
   echo "                          where DIR points to a previously used tmp directory"
   echo "                          (check for output 'Using DIR as tmp ...' of a previous"

From 990ebdae4dd77e396add2b2237efa1f6a70708d6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 08:57:54 +0100
Subject: [PATCH 10/78] move commands to install Apptainer in GitHub Actions
 Ubuntu VM to a script

---
 .github/workflows/tests_scripts.yml | 11 +----------
 install_apptainer_ubuntu.sh         | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 10 deletions(-)
 create mode 100755 install_apptainer_ubuntu.sh

diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 9c4975c381..ca23a15c40 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -27,18 +27,9 @@ jobs:
     - name: checkout
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-    # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
     - name: install Apptainer
       run: |
-        sudo apt-get install alien
-        alien --version
-        apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
-        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/$apptainer_rpm
-        sudo alien -d $apptainer_rpm
-        sudo apt install ./apptainer*.deb
-        apptainer --version
-        # also check whether 'singularity' command is still provided by Apptainer installation
-        singularity --version
+        ./install_apptainer_ubuntu.sh
 
     - name: test install_software_layer.sh script
       run: |
diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh
new file mode 100755
index 0000000000..c35c34cda6
--- /dev/null
+++ b/install_apptainer_ubuntu.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
+sudo apt-get install alien
+alien --version
+apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
+curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm}
+sudo alien -d ${apptainer_rpm}
+sudo apt install ./apptainer*.deb
+apptainer --version
+# also check whether 'singularity' command is still provided by Apptainer installation
+singularity --version

From 3d143d853dbecffb746d5ab3610d188d4ca12bda Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 08:58:00 +0100
Subject: [PATCH 11/78] add GitHub Actions workflow for testing
 eessi_container.sh script

---
 .../workflows/test_eessi_container_script.yml | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 .github/workflows/test_eessi_container_script.yml

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
new file mode 100644
index 0000000000..c02cd0dacf
--- /dev/null
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -0,0 +1,28 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for eessi_container.sh script
+on: [push, pull_request, workflow_dispatch]
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  eessi_container_script:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        SCRIPT_ARGS:
+        - "--help"
+        - "--info"
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+        - name: Mount EESSI CernVM-FS pilot repository
+          uses: eessi/github-action-eessi@v2
+
+        - name: install Apptainer
+          run: |
+            ./install_apptainer_ubuntu.sh
+
+        - name: Test eessi_container.sh script
+          run: |
+              ./eessi_container.sh ${{matrix.SCRIPT_ARGS}}

From 5fe467a04cb188f47f2b52c0e7159ef64a38ab58 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 09:13:03 +0100
Subject: [PATCH 12/78] stop script as soon as any command does non-zero exit +
 undefined env vars are errors

---
 eessi_container.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index b6eb3894ef..c09ea75ebf 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -12,6 +12,10 @@
 # license: GPLv2
 #
 
+# -e: stop script as soon as any command has non-zero exit code
+# -u: treat usage of undefined variables as errors
+set -e -u
+
 # script overview
 # -. initial settings & exit codes
 # 0. parse args

From 57b831526fadf55ca1f886fd92efc11d75aa93f4 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 09:24:42 +0100
Subject: [PATCH 13/78] fix typo

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index c09ea75ebf..283c41ddc0 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -318,7 +318,7 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 mkdir -p ${EESSI_TMPDIR}/cfg
 if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
   # need to source defaults as late as possible (see other sourcing below)
-  source ${base_dir}/init/eessi_defaults
+  source ${TOPDIR}/init/eessi_defaults
 
   # strip "/cvmfs/" from default setting
   repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//}
@@ -383,7 +383,7 @@ else
   export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
   export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
   # need to source defaults as late as possible (after *_OVERRIDEs)
-  source ${base_dir}/init/eessi_defaults
+  source ${TOPDIR}/init/eessi_defaults
 fi
 
 

From fba368d19686c1f2edaeb524d0a78f86673b453e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 09:25:20 +0100
Subject: [PATCH 14/78] comment out 'set -e -u' for now

---
 eessi_container.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 283c41ddc0..a63eb3ad44 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -14,7 +14,8 @@
 
 # -e: stop script as soon as any command has non-zero exit code
 # -u: treat usage of undefined variables as errors
-set -e -u
+# FIXME commented out because it's OK (?) if some environment variables are not set (like $SINGULARITY_HOME)
+# set -e -u
 
 # script overview
 # -. initial settings & exit codes

From 09feff74d11d4cf14393cde33abd0672eb9f3bcf Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 10:27:05 +0100
Subject: [PATCH 15/78] rename --info to --verbose

---
 .../workflows/test_eessi_container_script.yml |  2 +-
 eessi_container.sh                            | 34 +++++++++----------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index c02cd0dacf..a940a86924 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -11,7 +11,7 @@ jobs:
       matrix:
         SCRIPT_ARGS:
         - "--help"
-        - "--info"
+        - "--verbose"
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
diff --git a/eessi_container.sh b/eessi_container.sh
index a63eb3ad44..166f5ff2a4 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -92,7 +92,7 @@ display_help() {
   echo " FEATURES/OPTIONS to be implemented:"
   echo "  -d | --dry-run           -  run script except for executing the container,"
   echo "                              print information about setup [default: false]"
-  echo "  -i | --info              -  display configured repositories [default: false]"
+  echo "  -v | --verbose           -  display more information [default: false]"
   #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
   #echo "                              [default: not set]"
   #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
@@ -103,7 +103,7 @@ display_help() {
 ACCESS="ro"
 CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 DRY_RUN=0
-INFO=0
+VERBOSE=0
 STORAGE=
 MODE="shell"
 REPOSITORY="EESSI-pilot"
@@ -137,10 +137,6 @@ while [[ $# -gt 0 ]]; do
       display_help
       exit 0
       ;;
-    -i|--info)
-      INFO=1
-      shift 1
-      ;;
     -m|--mode)
       MODE="$2"
       shift 2
@@ -157,6 +153,10 @@ while [[ $# -gt 0 ]]; do
       RESUME="$2"
       shift 2
       ;;
+    -v|--verbose)
+      VERBOSE=1
+      shift 1
+      ;;
 #    -x|--http-proxy)
 #      HTTP_PROXY="$2"
 #      export http_proxy=${HTTP_PROXY}
@@ -261,7 +261,7 @@ else
     # mktemp falls back to using /tmp if TMPDIR is empty
     # TODO check if /tmp is writable, large enough and usable (different
     #      features for ro-access and rw-access)
-    [[ ${INFO} -eq 1 ]] && echo "skipping sanity checks for /tmp"
+    [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp"
   fi
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
   echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
@@ -287,32 +287,32 @@ fi
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
-[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
+[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
 # configure Singularity
 export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
 mkdir -p ${SINGULARITY_CACHEDIR}
-[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
+[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}
 EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN}
 mkdir -p ${EESSI_CVMFS_VAR_LIB}
 mkdir -p ${EESSI_CVMFS_VAR_RUN}
-[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}"
-[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}"
+[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}"
+[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}"
 
 # allow that SINGULARITY_HOME is defined before script is run
 if [[ -z ${SINGULARITY_HOME} ]]; then
   export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
   mkdir -p ${EESSI_TMPDIR}/home
-  [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
+  [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
-[[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
+[[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
 # set up repository config (always create cfg dir and populate it with info when
 # arg -r|--repository is used)
@@ -362,7 +362,7 @@ else
 
   # convert config_map into associative array cfg_file_map
   cfg_init_file_map "${config_map}"
-  [[ ${INFO} -eq 1 ]] && cfg_print_map
+  [[ ${VERBOSE} -eq 1 ]] && cfg_print_map
 
   # TODO use information to set up dir ${EESSI_TMPDIR}/cfg,
   #      define BIND mounts and override repo name and version
@@ -403,8 +403,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
   EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
-  [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
+  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
+  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
 
   # set environment variables for fuse mounts in Singularity container
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
@@ -433,7 +433,7 @@ if [[ -z ${SINGULARITY_BIND} ]]; then
 else
     export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}"
 fi
-[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
+[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
 
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"

From fe83f85fe93bdb492946355ce611fcc96906cf8f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 11:07:58 +0100
Subject: [PATCH 16/78] pin eessi/github-action-eessi by hash

---
 .github/workflows/test_eessi_container_script.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index a940a86924..fc4cd01f48 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -17,7 +17,7 @@ jobs:
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
         - name: Mount EESSI CernVM-FS pilot repository
-          uses: eessi/github-action-eessi@v2
+          uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
 
         - name: install Apptainer
           run: |

From 125bfa25d30e849e783bacb68c16200bea75dd7e Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 11:23:38 +0100
Subject: [PATCH 17/78] use exit code of container command in
 eessi_container.sh

---
 eessi_container.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index 166f5ff2a4..81ad7b4b7a 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -440,6 +440,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
 singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
+exit_code=$?
 
 # 7. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
@@ -461,3 +462,6 @@ if [[ ! -z ${SAVE} ]]; then
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)
+
+# use exit code of container command
+exit ${exit_code}

From be3e39909797986845f2be4274ee5c4375c2ae7b Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 13:01:21 +0100
Subject: [PATCH 18/78] replace RUN_SCRIPT_AND_ARGS with

---
 eessi_container.sh | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 166f5ff2a4..4d61677e46 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -111,7 +111,6 @@ RESUME=
 SAVE=
 #HTTP_PROXY=
 #HTTPS_PROXY=
-RUN_SCRIPT_AND_ARGS=
 
 POSITIONAL_ARGS=()
 
@@ -220,8 +219,6 @@ fi
 if [[ "${MODE}" == "run" ]]; then
   if [[ $# -eq 0 ]]; then
     fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}"
-  else
-    RUN_SCRIPT_AND_ARGS=$@
   fi
 fi
 
@@ -436,10 +433,10 @@ fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
 
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}"
+echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
-singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}
+singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 
 # 7. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then

From 27cbdf3e0c3507925148945e41dc0683772489c6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 15:52:49 +0100
Subject: [PATCH 19/78] add checks for various use cases of eessi_container.sh
 script (+ no need to natively mount EESSI, since that's done in the
 container)

---
 .../workflows/test_eessi_container_script.yml | 86 +++++++++++++++++--
 1 file changed, 79 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index fc4cd01f48..d1b6b1cb74 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -9,20 +9,92 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        SCRIPT_ARGS:
-        - "--help"
-        - "--verbose"
+        SCRIPT_TEST:
+        - help
+        - run
+        - shell
+        - container
+        - readwrite
+        - resume
+        - save
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-        - name: Mount EESSI CernVM-FS pilot repository
-          uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
-
         - name: install Apptainer
           run: |
             ./install_apptainer_ubuntu.sh
 
         - name: Test eessi_container.sh script
           run: |
-              ./eessi_container.sh ${{matrix.SCRIPT_ARGS}}
+              test_cmd="cat /etc/os-release"
+              out_pattern="Debian GNU/Linux 11"
+
+              if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
+                ./eessi_container.sh --help
+
+              # test use of --mode run
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
+                outfile=out_run.txt
+                echo "${test_cmd}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --mode run /test/test_script.sh | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --mode shell
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then
+                outfile=out_shell.txt
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --container option, using a totally different container;
+              # cfr. https://github.com/easybuilders/easybuild-containers
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then
+                outfile=out_container.txt
+                container="docker://ghcr.io/eessi/build-node:debian10"
+                ./eessi_container.sh --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of '--access rw' to get write access in container
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
+                outfile=out_readwrite.txt
+                fn="test_${RANDOM}.txt"
+                echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --access rw --mode run /test/test_script.sh > ${outfile}
+
+                tmpdir=$(grep "--resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                # note: must use '--access rw' again here, since touched file is in overlay upper dir
+                ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+
+              # test use of --resume
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
+                outfile=out_resume.txt
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                tmpdir=$(grep "--resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --save (+ --resume)
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
+                outfile=out_save.txt
+                fn="test_${RANDOM}.txt"
+                test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
+                ./eessi_container.sh --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+
+                tar tfv test-save.tar | grep "overlay-upper/${fn}"
+
+              else
+                  echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2
+                  exit 1
+              fi

From b34afca53a24c72ba69f23956e1d55c7a5cc6585 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 16:18:08 +0100
Subject: [PATCH 20/78] commented dry-run option, moved verbose option in help
 msg

---
 eessi_container.sh | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e0f4ead1af..96a9900a5e 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -86,13 +86,13 @@ display_help() {
   echo "                          when a directory is provided, the format of the"
   echo "                          tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
   echo "                          [default: not set]"
+  echo "  -v | --verbose        - display more information [default: false]"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
-  echo
-  echo " FEATURES/OPTIONS to be implemented:"
-  echo "  -d | --dry-run           -  run script except for executing the container,"
-  echo "                              print information about setup [default: false]"
-  echo "  -v | --verbose           -  display more information [default: false]"
+  #echo
+  #echo " FEATURES/OPTIONS to be implemented:"
+  #echo "  -d | --dry-run           -  run script except for executing the container,"
+  #echo "                              print information about setup [default: false]"
   #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
   #echo "                              [default: not set]"
   #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
@@ -102,7 +102,7 @@ display_help() {
 # set defaults for command line arguments
 ACCESS="ro"
 CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
-DRY_RUN=0
+#DRY_RUN=0
 VERBOSE=0
 STORAGE=
 MODE="shell"
@@ -124,10 +124,10 @@ while [[ $# -gt 0 ]]; do
       CONTAINER="$2"
       shift 2
       ;;
-    -d|--dry-run)
-      DRY_RUN=1
-      shift 1
-      ;;
+#    -d|--dry-run)
+#      DRY_RUN=1
+#      shift 1
+#      ;;
     -g|--storage)
       STORAGE="$2"
       shift 2

From 7d3818d539111186f7d33dbf1b3a34813cd4e58b Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 15:52:49 +0100
Subject: [PATCH 21/78] add checks for various use cases of eessi_container.sh
 script (+ no need to natively mount EESSI, since that's done in the
 container)

---
 .../workflows/test_eessi_container_script.yml | 86 +++++++++++++++++--
 1 file changed, 79 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index fc4cd01f48..362bd3893c 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -9,20 +9,92 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        SCRIPT_ARGS:
-        - "--help"
-        - "--verbose"
+        SCRIPT_TEST:
+        - help
+        - run
+        - shell
+        - container
+        - readwrite
+        - resume
+        - save
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-        - name: Mount EESSI CernVM-FS pilot repository
-          uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
-
         - name: install Apptainer
           run: |
             ./install_apptainer_ubuntu.sh
 
         - name: Test eessi_container.sh script
           run: |
-              ./eessi_container.sh ${{matrix.SCRIPT_ARGS}}
+              test_cmd="cat /etc/os-release"
+              out_pattern="Debian GNU/Linux 11"
+
+              if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
+                ./eessi_container.sh --help
+
+              # test use of --mode run
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
+                outfile=out_run.txt
+                echo "${test_cmd}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --mode run /test/test_script.sh | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --mode shell
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then
+                outfile=out_shell.txt
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --container option, using a totally different container;
+              # cfr. https://github.com/easybuilders/easybuild-containers
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then
+                outfile=out_container.txt
+                container="docker://ghcr.io/eessi/build-node:debian10"
+                ./eessi_container.sh --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "Debian GNU/Linux 10" ${outfile}
+
+              # test use of '--access rw' to get write access in container
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
+                outfile=out_readwrite.txt
+                fn="test_${RANDOM}.txt"
+                echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --access rw --mode run /test/test_script.sh > ${outfile}
+
+                tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                # note: must use '--access rw' again here, since touched file is in overlay upper dir
+                ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+
+              # test use of --resume
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
+                outfile=out_resume.txt
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --save (+ --resume)
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
+                outfile=out_save.txt
+                fn="test_${RANDOM}.txt"
+                test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
+                ./eessi_container.sh --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+
+                tar tfv test-save.tar | grep "overlay-upper/${fn}"
+
+              else
+                  echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2
+                  exit 1
+              fi

From 602aa68068b275b56b181af9bbcbf12f8395eedb Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Wed, 8 Feb 2023 16:28:52 +0100
Subject: [PATCH 22/78] fix check for './eessi_container.sh --resume'

---
 .github/workflows/test_eessi_container_script.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 362bd3893c..4d1c3bf898 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -73,11 +73,13 @@ jobs:
               # test use of --resume
               elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
                 outfile=out_resume.txt
-                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
-                rm -f ${outfile}
+                ./eessi_container.sh --mode shell <<< "${test_cmd}" > ${outfile}
 
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                rm -f ${outfile}
+
                 ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                cat ${outfile}
                 grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
                 grep "${out_pattern}" ${outfile}
 

From adcc655f99c65a609bc1cf21f1fe9d6fc89a8c7c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 8 Feb 2023 17:01:26 +0100
Subject: [PATCH 23/78] fix small typos in grep commands

---
 .github/workflows/test_eessi_container_script.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 362bd3893c..3974b42506 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -68,7 +68,7 @@ jobs:
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 # note: must use '--access rw' again here, since touched file is in overlay upper dir
                 ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
 
               # test use of --resume
               elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
@@ -90,7 +90,7 @@ jobs:
                 rm -f ${outfile}
 
                 ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
 
                 tar tfv test-save.tar | grep "overlay-upper/${fn}"
 

From 7e470233684d456830f9cb7559992a7140845a57 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 10 Feb 2023 09:11:46 +0100
Subject: [PATCH 24/78] enable --verbose + add step to collect test env info

---
 .../workflows/test_eessi_container_script.yml | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index c8c2aef47e..0920678c24 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -25,6 +25,11 @@ jobs:
           run: |
             ./install_apptainer_ubuntu.sh
 
+        - name: Collect info on test environment
+          run: |
+              mount
+              df -h
+
         - name: Test eessi_container.sh script
           run: |
               test_cmd="cat /etc/os-release"
@@ -39,13 +44,13 @@ jobs:
                 echo "${test_cmd}" > test_script.sh
                 chmod u+x test_script.sh
                 export SINGULARITY_BIND="$PWD:/test"
-                ./eessi_container.sh --mode run /test/test_script.sh | tee ${outfile}
+                ./eessi_container.sh --verbose --mode run /test/test_script.sh | tee ${outfile}
                 grep "${out_pattern}" ${outfile}
 
               # test use of --mode shell
               elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then
                 outfile=out_shell.txt
-                ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
                 grep "${out_pattern}" ${outfile}
 
               # test use of --container option, using a totally different container;
@@ -53,7 +58,7 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then
                 outfile=out_container.txt
                 container="docker://ghcr.io/eessi/build-node:debian10"
-                ./eessi_container.sh --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                ./eessi_container.sh --verbose --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
                 grep "Debian GNU/Linux 10" ${outfile}
 
               # test use of '--access rw' to get write access in container
@@ -63,22 +68,22 @@ jobs:
                 echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
                 chmod u+x test_script.sh
                 export SINGULARITY_BIND="$PWD:/test"
-                ./eessi_container.sh --access rw --mode run /test/test_script.sh > ${outfile}
+                ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}
 
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 # note: must use '--access rw' again here, since touched file is in overlay upper dir
-                ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
                 grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
 
               # test use of --resume
               elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
                 outfile=out_resume.txt
-                ./eessi_container.sh --mode shell <<< "${test_cmd}" > ${outfile}
+                ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" > ${outfile}
 
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 rm -f ${outfile}
 
-                ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
                 cat ${outfile}
                 grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
                 grep "${out_pattern}" ${outfile}
@@ -88,10 +93,10 @@ jobs:
                 outfile=out_save.txt
                 fn="test_${RANDOM}.txt"
                 test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
-                ./eessi_container.sh --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
                 rm -f ${outfile}
 
-                ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
                 grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
 
                 tar tfv test-save.tar | grep "overlay-upper/${fn}"

From a1bb7df72857be55e7e626de09634d1015cb45c0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 10 Feb 2023 09:35:56 +0100
Subject: [PATCH 25/78] disable broken tests for eessi_container.sh script for
 now

---
 .github/workflows/test_eessi_container_script.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 0920678c24..7678aedc64 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -14,9 +14,10 @@ jobs:
         - run
         - shell
         - container
-        - readwrite
         - resume
-        - save
+        # FIXME disabled because '--access rw' is not working in CI environment
+        #- readwrite
+        #- save
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0

From b1a36355de66fa9a9557edbef79508f88c763cbb Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 24 Jan 2023 21:59:41 +0100
Subject: [PATCH 26/78] added a function to obtain full path to a tool

---
 scripts/utils.sh | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index 5d8455bb68..cd7f758a0d 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -31,3 +31,35 @@ function check_exit_code {
         fatal_error "${fail_msg}"
     fi
 }
+
+function get_path_for_tool {
+    tool_name=$1
+    tool_envvar_name=$2
+
+    which_out=$(which ${tool_name} 2>&1)
+    exit_code=$?
+    if [[ ${exit_code} -eq 0 ]]; then
+        echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2
+        echo "${which_out}"
+        return 0
+    fi
+    if [[ -z "${tool_envvar_name}" ]]; then
+        msg="no env var holding the full path to tool '${tool_name}' provided"
+        echo "${msg}" >&2
+        return 1
+    else
+        tool_envvar_value=${!tool_envvar_name}
+        if [[ -x "${tool_envvar_value}" ]]; then
+            msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}"
+            echo "${msg}" >&2
+            echo "${tool_envvar_value}"
+            return 0
+        else
+            msg="ERROR: tool '${tool_name}' not in PATH\n"
+            msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH"
+            echo "${msg}" >&2
+            echo ""
+            return 2
+        fi
+    fi
+}

From 27bc2035b00f59442fd46bcdbd6e104c09944f18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Wed, 25 Jan 2023 19:37:55 +0100
Subject: [PATCH 27/78] added function to determine IPv4 address

---
 scripts/utils.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index cd7f758a0d..db533adc12 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -63,3 +63,12 @@ function get_path_for_tool {
         fi
     fi
 }
+
+function get_ipv4_address {
+    hname=$1
+    hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
+    # TODO try other methods if the one above does not work --> tool that verifies
+    #      what method can be used?
+    echo "${hipv4}"
+    return 0
+}

From bfe6d36bf056c9e4e9719fa3a8110899fb125dc2 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 20:44:16 +0100
Subject: [PATCH 28/78] various updates for bot/build.sh and eessi_container.sh

cherry-picked via commit b5bf008e4136b7877e06ca0602b6fd6aa49128c0
ONLY picked chages to eessi_container.sh
---
 eessi_container.sh | 103 ++++++++++++++++++++++++---------------------
 1 file changed, 55 insertions(+), 48 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 96a9900a5e..3045fd5a89 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -43,8 +43,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5))
 REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6))
 RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7))
 SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8))
-#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
-#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
+HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
+HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 
 # CernVM-FS settings
@@ -64,39 +64,39 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
 display_help() {
   echo "usage: $0 [OPTIONS] [SCRIPT]"
   echo " OPTIONS:"
-  echo "  -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]"
-  echo "  -c | --container IMG  - image file or URL defining the container to use"
-  echo "                          [default: docker://ghcr.io/eessi/build-node:debian11]"
-  echo "  -h | --help           - display this usage information [default: false]"
-  echo "  -g | --storage DIR    - directory space on host machine (used for"
-  echo "                          temporary data) [default: 1. TMPDIR, 2. /tmp]"
-  echo "  -m | --mode MODE      - with MODE==shell (launch interactive shell) or"
-  echo "                          MODE==run (run a script) [default: shell]"
-  echo "  -r | --repository CFG - configuration file or identifier defining the"
-  echo "                          repository to use [default: EESSI-pilot via"
+  echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
+  echo "  -c | --container IMG   - image file or URL defining the container to use"
+  echo "                           [default: docker://ghcr.io/eessi/build-node:debian11]"
+  echo "  -h | --help            - display this usage information [default: false]"
+  echo "  -g | --storage DIR     - directory space on host machine (used for"
+  echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
+  echo "                           MODE==run (run a script) [default: shell]"
+  echo "  -r | --repository CFG  - configuration file or identifier defining the"
+  echo "                           repository to use [default: EESSI-pilot via"
   echo "                          container configuration]"
-  echo "  -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
-  echo "                          where DIR points to a previously used tmp directory"
-  echo "                          (check for output 'Using DIR as tmp ...' of a previous"
-  echo "                          run) and TGZ is the path to a tarball which is"
-  echo "                          unpacked the tmp dir stored on the local storage space"
-  echo "                          (see option --storage above) [default: not set]"
-  echo "  -s | --save DIR/TGZ   - save contents of tmp directory to a tarball in"
-  echo "                          directory DIR or provided with the fixed full path TGZ"
-  echo "                          when a directory is provided, the format of the"
-  echo "                          tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
-  echo "                          [default: not set]"
-  echo "  -v | --verbose        - display more information [default: false]"
+  echo "  -u | --resume DIR/TGZ  - resume a previous run from a directory or tarball,"
+  echo "                           where DIR points to a previously used tmp directory"
+  echo "                           (check for output 'Using DIR as tmp ...' of a previous"
+  echo "                           run) and TGZ is the path to a tarball which is"
+  echo "                           unpacked the tmp dir stored on the local storage space"
+  echo "                           (see option --storage above) [default: not set]"
+  echo "  -s | --save DIR/TGZ    - save contents of tmp directory to a tarball in"
+  echo "                           directory DIR or provided with the fixed full path TGZ"
+  echo "                           when a directory is provided, the format of the"
+  echo "                           tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz"
+  echo "                           [default: not set]"
+  echo "  -v | --verbose         - display more information [default: false]"
+  echo "  -x | --http-proxy URL  - provides URL for the env variable http_proxy"
+  echo "                           [default: not set]; uses env var \$http_proxy if set"
+  echo "  -y | --https-proxy URL - provides URL for the env variable https_proxy"
+  echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
-  #echo
-  #echo " FEATURES/OPTIONS to be implemented:"
-  #echo "  -d | --dry-run           -  run script except for executing the container,"
-  #echo "                              print information about setup [default: false]"
-  #echo "  -x | --http-proxy URL    -  provides URL for the env variable http_proxy"
-  #echo "                              [default: not set]"
-  #echo "  -y | --https-proxy URL   -  provides URL for the env variable https_proxy"
-  #echo "                              [default: not set]"
+  echo
+  echo " FEATURES/OPTIONS to be implemented:"
+  echo "  -d | --dry-run         -  run script except for executing the container,"
+  echo "                            print information about setup [default: false]"
 }
 
 # set defaults for command line arguments
@@ -109,8 +109,8 @@ MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
 SAVE=
-#HTTP_PROXY=
-#HTTPS_PROXY=
+HTTP_PROXY=${http_proxy:-}
+HTTPS_PROXY=${https_proxy:-}
 
 POSITIONAL_ARGS=()
 
@@ -156,16 +156,16 @@ while [[ $# -gt 0 ]]; do
       VERBOSE=1
       shift 1
       ;;
-#    -x|--http-proxy)
-#      HTTP_PROXY="$2"
-#      export http_proxy=${HTTP_PROXY}
-#      shift 2
-#      ;;
-#    -y|--https-proxy)
-#      HTTPS_PROXY="$2"
-#      export https_proxy=${HTTPS_PROXY}
-#      shift 2
-#      ;;
+    -x|--http-proxy)
+      HTTP_PROXY="$2"
+      export http_proxy=${HTTP_PROXY}
+      shift 2
+      ;;
+    -y|--https-proxy)
+      HTTPS_PROXY="$2"
+      export https_proxy=${HTTPS_PROXY}
+      shift 2
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;
@@ -311,9 +311,9 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru
 BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
 [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
-# set up repository config (always create cfg dir and populate it with info when
+# set up repository config (always create directory repos_cfg and populate it with info when
 # arg -r|--repository is used)
-mkdir -p ${EESSI_TMPDIR}/cfg
+mkdir -p ${EESSI_TMPDIR}/repos_cfg
 if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
   # need to source defaults as late as possible (see other sourcing below)
   source ${TOPDIR}/init/eessi_defaults
@@ -370,13 +370,13 @@ else
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg
+    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
   do
     target=${cfg_file_map[${src}]}
-    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}"
+    BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}"
   done
   export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
   export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
@@ -432,6 +432,13 @@ else
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}"
 
+# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set)
+if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+    export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+    # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility)
+    export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+fi
+
 echo "Launching container with command (next line):"
 echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level

From 837234107d28cd9b2d759a66af8f7e2ec5244923 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 20:55:30 +0100
Subject: [PATCH 29/78] provide CVMFS_HTTP_PROXY if necessary

cherry-picked via commit b5c07eeb99e77191d818e64788b0d419031cf526
NOTE, only applied changes to eessi_container.sh
---
 eessi_container.sh | 39 ++++++++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 3045fd5a89..9238d3c793 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -24,8 +24,8 @@
 # 2. set up host storage/tmp
 # 3. set up common vars and directories
 # 4. set up vars specific to a scenario
-# 5. initialize host storage/tmp from previous run if provided
-# 6. run container
+# 5. run container
+# 6. save tmp (if requested)
 
 # -. initial settings & exit codes
 TOPDIR=$(dirname $(realpath $0))
@@ -279,7 +279,7 @@ fi
 #      |-overlay-upper
 #      |-overlay-work
 #      |-home
-#      |-cfg
+#      |-repos_cfg
 
 # tmp dir for EESSI
 EESSI_TMPDIR=${EESSI_HOST_STORAGE}
@@ -361,9 +361,9 @@ else
   cfg_init_file_map "${config_map}"
   [[ ${VERBOSE} -eq 1 ]] && cfg_print_map
 
-  # TODO use information to set up dir ${EESSI_TMPDIR}/cfg,
-  #      define BIND mounts and override repo name and version
-  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg
+  # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
+  #     define BIND mounts and override repo name and version
+  # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
   if [[ ! -r ${config_bundle} ]]; then
     fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
@@ -384,6 +384,26 @@ else
   source ${TOPDIR}/init/eessi_defaults
 fi
 
+# if http_proxy is not empty, we assume that the machine accesses internet
+# via a proxy. then we need to add CVMFS_HTTP_PROXY to
+# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND
+# MOUNT if it was not yet in BIND_PATHS)
+if [[ ! -z ${http_proxy} ]]; then
+    # TODO tolerate other formats for proxy URLs, for now assume format is
+    # http://SOME_HOSTNAME:SOME_PORT
+    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
+    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
+    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
+         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+       >> ${EESSI_TMPDIR}/repos_cfg/default.local
+    cat ${EESSI_TMPDIR}/repos_cfg/default.local
+    # if default.local is not BIND mounted into container, add it to BIND_PATHS
+    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+        export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
+    fi
+fi
 
 # 4. set up vars and dirs specific to a scenario
 
@@ -420,10 +440,7 @@ if [[ "${ACCESS}" == "rw" ]]; then
 fi
 
 
-# 5. initialize host storage/tmp from previous run if provided
-
-
-# 6. run container
+# 5. run container
 # final settings
 if [[ -z ${SINGULARITY_BIND} ]]; then
     export SINGULARITY_BIND="${BIND_PATHS}"
@@ -446,7 +463,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 
-# 7. save tmp if requested (arg -s|--save)
+# 6. save tmp if requested (arg -s|--save)
 if [[ ! -z ${SAVE} ]]; then
   # Note, for now we don't try to be smart and record in any way the OS and
   #   ARCH which might have been used internally, eg, when software packages

From 3f1aed9a2b5da7fa7e017e415d24a412dbfa18fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:41:36 +0100
Subject: [PATCH 30/78] add functions to derive hostname and port number from
 proxy URL

---
 scripts/utils.sh | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index db533adc12..fb4eff1e3f 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -64,6 +64,30 @@ function get_path_for_tool {
     fi
 }
 
+function get_host_from_url {
+    url=$1
+    re="(http|https)://([^/:]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
+function get_port_from_url {
+    url=$1
+    re="(http|https)://[^:]+:([0-9]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi 
+}
+
 function get_ipv4_address {
     hname=$1
     hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)

From b7abfc6e7de85e2b46e3241d8946a1211dcde568 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Fri, 27 Jan 2023 22:43:27 +0100
Subject: [PATCH 31/78] add function to determine config sections

---
 cfg_files.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index 885ebd0877..ffd29db50a 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -47,9 +47,7 @@ function cfg_load {
     else
       val=$(cfg_get_key_value $line)
       # trim leading and trailing spaces as well
-      #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces)
-      #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//')
       cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces)
       if [[ -n "$cur_key" ]]; then
         # section + key is the associative in bash array, the field separator is space
@@ -69,6 +67,20 @@ function cfg_print {
   done
 }
 
+function cfg_sections {
+  declare -A sections
+  for key in "${!cfg_repos[@]}"
+  do
+    # extract section from the associative key
+    section=$(echo $key | cut -f1 -d ' ')
+    sections[${section}]=1
+  done
+  for repo in "${!sections[@]}"
+  do
+      echo "${repo}"
+  done
+}
+
 function cfg_get_value {
   section=$1
   key=$2

From 9b7b8a4db00261b0fe2bd40152d7adacde11ffe3 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:17:11 +0100
Subject: [PATCH 32/78] add parameter to list repositories, fix bugs

fixed bugs

- determining IP address for proxy host was wrong
- handling of positional parameters for executing commands with arguments was wrong
---
 eessi_container.sh | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 9238d3c793..06b77a5acf 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -70,6 +70,7 @@ display_help() {
   echo "  -h | --help            - display this usage information [default: false]"
   echo "  -g | --storage DIR     - directory space on host machine (used for"
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
   echo "                           MODE==run (run a script) [default: shell]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
@@ -105,6 +106,7 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 #DRY_RUN=0
 VERBOSE=0
 STORAGE=
+LIST_REPOS=0
 MODE="shell"
 REPOSITORY="EESSI-pilot"
 RESUME=
@@ -136,6 +138,10 @@ while [[ $# -gt 0 ]]; do
       display_help
       exit 0
       ;;
+    -l|--list-repos)
+      LIST_REPOS=1
+      shift 1
+      ;;
     -m|--mode)
       MODE="$2"
       shift 2
@@ -178,6 +184,17 @@ done
 
 set -- "${POSITIONAL_ARGS[@]}"
 
+if [[ ${LIST_REPOS} -eq 1 ]]; then
+    echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':"
+    echo "    EESSI-pilot [default]"
+    cfg_load ${EESSI_REPOS_CFG_FILE}
+    sections=$(cfg_sections)
+    while IFS= read -r repo_id
+    do
+        echo "    ${repo_id}"
+    done <<< "${sections}"
+    exit 0
+fi
 
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
@@ -390,17 +407,20 @@ fi
 # MOUNT if it was not yet in BIND_PATHS)
 if [[ ! -z ${http_proxy} ]]; then
     # TODO tolerate other formats for proxy URLs, for now assume format is
-    # http://SOME_HOSTNAME:SOME_PORT
-    PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http://
-    PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port
-    HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/}
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) 
-    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \
-         "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
+    # http://SOME_HOSTNAME:SOME_PORT/
+    [[ ${VERBOSE} -eq 1 ]] && echo "http_proxy='${http_proxy}'"
+    PROXY_HOST=$(get_host_from_url ${http_proxy})
+    [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
+    PROXY_PORT=$(get_port_from_url ${http_proxy})
+    [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
+    echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
     cat ${EESSI_TMPDIR}/repos_cfg/default.local
+
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
-    if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
+    if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
         export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local"
     fi
 fi

From 4a414f593d6ff02629d1374e67006dcd1a85f71c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:26:19 +0100
Subject: [PATCH 33/78] replace FILE with DIR in EESSI_REPOS_CFG_FILE_OVERRIDE

NOTE only applied changes to eessi_container.sh
---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 06b77a5acf..d59bb38687 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -52,8 +52,8 @@ CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
 # repository cfg file, default name (default location: $PWD)
-#   can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}"
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
 
 
 # 0. parse args

From 37abf4f8aab9d138ad763cd457d2476bef16a6aa Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 29 Jan 2023 19:56:41 +0100
Subject: [PATCH 34/78] fix whitespace issues

---
 cfg_files.sh        | 1 -
 eessi_container.sh  | 2 +-
 init/eessi_defaults | 1 -
 scripts/utils.sh    | 4 ++--
 4 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/cfg_files.sh b/cfg_files.sh
index ffd29db50a..57ea2f7c03 100644
--- a/cfg_files.sh
+++ b/cfg_files.sh
@@ -165,4 +165,3 @@ function cfg_print_map {
     echo "${index} --> ${cfg_file_map[${index}]}"
   done
 }
-
diff --git a/eessi_container.sh b/eessi_container.sh
index d59bb38687..1a2e61227b 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -413,7 +413,7 @@ if [[ ! -z ${http_proxy} ]]; then
     [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'"
     PROXY_PORT=$(get_port_from_url ${http_proxy})
     [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'"
-    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) 
+    HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST})
     [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
     echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
diff --git a/init/eessi_defaults b/init/eessi_defaults
index 1b5ce07fb1..f482cbc269 100644
--- a/init/eessi_defaults
+++ b/init/eessi_defaults
@@ -10,4 +10,3 @@
 
 export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}"
 export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}"
-
diff --git a/scripts/utils.sh b/scripts/utils.sh
index fb4eff1e3f..d0da95e87f 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -73,7 +73,7 @@ function get_host_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_port_from_url {
@@ -85,7 +85,7 @@ function get_port_from_url {
     else
         echo ""
         return 1
-    fi 
+    fi
 }
 
 function get_ipv4_address {

From d468a1ababa8072c4069917f4c16529cd362011f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:42:34 +0100
Subject: [PATCH 35/78] include changes made to test PR for building with the
 bot

cherry-picked via commit fce504fb466be005f1d417e96e47f442c8f29c52
NOTE, only changed eessi_container.sh
---
 eessi_container.sh | 50 ++++++++++++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 15 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 1a2e61227b..14fb716abb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -51,9 +51,15 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 CVMFS_VAR_LIB="var-lib-cvmfs"
 CVMFS_VAR_RUN="var-run-cvmfs"
 
+# directory for tmp used inside container
+export TMP_IN_CONTAINER=/tmp
+
 # repository cfg file, default name (default location: $PWD)
 #   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg"
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg"
+# other repository cfg files in directory, default location: $PWD
+#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}"
 
 
 # 0. parse args
@@ -199,7 +205,7 @@ fi
 # 1. check if argument values are valid
 # (arg -a|--access) check if ACCESS is supported
 if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then
-  fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
+    fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -c|--container) check container (is it a file or URL & access those)
@@ -211,11 +217,14 @@ fi
 
 # (arg -m|--mode) check if MODE is known
 if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then
-  fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
+    fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}"
 fi
 
 # TODO (arg -r|--repository) check if repository is known
 # REPOSITORY_ERROR_EXITCODE
+if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
+    fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}"
+fi
 
 # TODO (arg -u|--resume) check if it exists, if user has read permission,
 #      if it contains data from a previous run
@@ -249,6 +258,7 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then
   #   skip creating a new tmp directory, just set environment variables
   echo "Resuming from previous run using temporary storage at ${RESUME}"
   EESSI_HOST_STORAGE=${RESUME}
+  echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 else
   # we need a tmp location (and possibly init it with ${RESUME} if it was not
   #   a directory
@@ -325,7 +335,9 @@ fi
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"
-BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp"
+# provide a '/tmp' inside the container
+BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}"
+
 [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
 
 # set up repository config (always create directory repos_cfg and populate it with info when
@@ -381,13 +393,22 @@ else
   # use information to set up dir ${EESSI_TMPDIR}/repos_cfg,
   #     define BIND mounts and override repo name and version
   # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg
-  if [[ ! -r ${config_bundle} ]]; then
-    fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
+  # if config_bundle is relative path (no '/' at start) prepend it with
+  # EESSI_REPOS_CFG_DIR
+  config_bundle_path=
+  if [[ ! "${config_bundle}" =~ ^/ ]]; then
+      config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle}
+  else
+      config_bundle_path=${config_bundle}
+  fi
+
+  if [[ ! -r ${config_bundle_path} ]]; then
+    fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE}
   fi
 
   # only unpack config_bundle if we're not resuming from a previous run
   if [[ -z ${RESUME} ]]; then
-    tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg
+    tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg
   fi
 
   for src in "${!cfg_file_map[@]}"
@@ -436,12 +457,8 @@ if [[ "${ACCESS}" == "ro" ]]; then
 fi
 
 if [[ "${ACCESS}" == "rw" ]]; then
-  EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper
-  EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work
   mkdir -p ${EESSI_TMPDIR}/overlay-upper
   mkdir -p ${EESSI_TMPDIR}/overlay-work
-  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}"
-  [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}"
 
   # set environment variables for fuse mounts in Singularity container
   export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
@@ -450,8 +467,8 @@ if [[ "${ACCESS}" == "rw" ]]; then
 
   EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
   EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper"
-  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
+  EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
   EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
   export EESSI_PILOT_WRITABLE_OVERLAY
 
@@ -476,11 +493,13 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
+# if INFO is set to 1 (arg --info), add argument '-q'
+RUN_QUIET=${INFO:--q}
 echo "Launching container with command (next line):"
-echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
+echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
 #      provided to the script
-singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
+singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 
 # 6. save tmp if requested (arg -s|--save)
@@ -500,6 +519,7 @@ if [[ ! -z ${SAVE} ]]; then
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
   echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
+  echo "RESUME_FROM_TGZ ${TGZ}"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From af658ee9229f36eac972bef87a36bbb6b883ef48 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 3 Feb 2023 20:58:01 +0100
Subject: [PATCH 36/78] fix issues in eessi_container.sh

---
 eessi_container.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 14fb716abb..23118d0d72 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -258,7 +258,6 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then
   #   skip creating a new tmp directory, just set environment variables
   echo "Resuming from previous run using temporary storage at ${RESUME}"
   EESSI_HOST_STORAGE=${RESUME}
-  echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 else
   # we need a tmp location (and possibly init it with ${RESUME} if it was not
   #   a directory
@@ -290,6 +289,7 @@ else
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
   echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
 fi
+echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
 if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then
@@ -494,7 +494,12 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
 fi
 
 # if INFO is set to 1 (arg --info), add argument '-q'
-RUN_QUIET=${INFO:--q}
+if [[ -z ${INFO} ]]; then
+    RUN_QUIET='-q'
+else
+    RUN_QUIET=''
+fi
+
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level

From eb66e1c4361f458114d2186a846fa9fe56b1df16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= <thomas.roblitz@uib.no>
Date: Sat, 4 Feb 2023 15:07:23 +0100
Subject: [PATCH 37/78] fix RUN_QUIET setting

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 23118d0d72..89d4bd98f6 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -493,8 +493,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 1 (arg --info), add argument '-q'
-if [[ -z ${INFO} ]]; then
+# if INFO is set to 0 (no arg --info), add argument '-q'
+if [[ ${INFO} -eq 0 ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''

From 1cb4674978bee259253330d769ca220ad795202a Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 21:59:39 +0100
Subject: [PATCH 38/78] improvements to resuming job environment

cherry-picked via commit 23e773ce0d8c6504e1440629a4178318265b73d2
NOTE only applied changes to eessi_container.sh
---
 eessi_container.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 89d4bd98f6..e714f8e2d0 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -360,6 +360,9 @@ else
   #    standard EESSI repositories)
   cfg_load ${EESSI_REPOS_CFG_FILE}
 
+  # copy repos.cfg to job directory --> makes it easier to inspect the job
+  cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
+
   # cfg file should include: repo_name, repo_version, config_bundle,
   #   map { local_filepath -> container_filepath }
   #
@@ -493,8 +496,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 0 (no arg --info), add argument '-q'
-if [[ ${INFO} -eq 0 ]]; then
+# if INFO is set to 1 (arg --info), add argument '-q'
+if [[ -z ${INFO} ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''

From 0c465da3de12ed1bb7494651031d64d7f17928e4 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 7 Feb 2023 19:27:07 +0100
Subject: [PATCH 39/78] address requested change

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e714f8e2d0..e8698cd719 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -496,8 +496,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 1 (arg --info), add argument '-q'
-if [[ -z ${INFO} ]]; then
+# if INFO is set to 0 (no arg --info), add argument '-q'
+if [[ ${INFO} -eq 0 ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''

From 26ab98c1f1cc901d6905424c9fdb2f5c9395ed79 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 22:16:36 +0100
Subject: [PATCH 40/78] cleaning up leftovers after adding updates from NESSI

---
 eessi_container.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e8698cd719..e7ad625a15 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -100,10 +100,6 @@ display_help() {
   echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
   echo " If value for --mode is 'run', the SCRIPT provided is executed."
-  echo
-  echo " FEATURES/OPTIONS to be implemented:"
-  echo "  -d | --dry-run         -  run script except for executing the container,"
-  echo "                            print information about setup [default: false]"
 }
 
 # set defaults for command line arguments
@@ -496,8 +492,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if INFO is set to 0 (no arg --info), add argument '-q'
-if [[ ${INFO} -eq 0 ]]; then
+# if VERBOSE is set to 0 (no arg --verbose), add argument '-q'
+if [[ ${VERBOSE} -eq 0 ]]; then
     RUN_QUIET='-q'
 else
     RUN_QUIET=''
@@ -505,8 +501,6 @@ fi
 
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
-# TODO for now we run singularity with '-q' (quiet), later adjust this to the log level
-#      provided to the script
 singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"
 exit_code=$?
 

From a44088e4c33694c0047a6e5874df780e3c2ff15e Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 18 Feb 2023 22:21:42 +0100
Subject: [PATCH 41/78] moving cfg_files.sh to scripts dir

---
 eessi_container.sh                   | 2 +-
 cfg_files.sh => scripts/cfg_files.sh | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename cfg_files.sh => scripts/cfg_files.sh (100%)

diff --git a/eessi_container.sh b/eessi_container.sh
index e7ad625a15..6d3fbc0adc 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -31,7 +31,7 @@
 TOPDIR=$(dirname $(realpath $0))
 
 source ${TOPDIR}/scripts/utils.sh
-source ${TOPDIR}/cfg_files.sh
+source ${TOPDIR}/scripts/cfg_files.sh
 
 # exit codes: bitwise shift codes to allow for combination of exit codes
 # ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh
diff --git a/cfg_files.sh b/scripts/cfg_files.sh
similarity index 100%
rename from cfg_files.sh
rename to scripts/cfg_files.sh

From bbbad0dce469299e4168a54a939facc4c1be7c60 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 20 Feb 2023 10:07:03 +0100
Subject: [PATCH 42/78] improved handling of container cache + explicit pull of
 image

cherry-picked via commit bfb1b29103dc309a12573f3ec9c247d432f08f73
Note, only part for eessi_container.sh included here
---
 eessi_container.sh | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 6d3fbc0adc..cdd2c017fb 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -310,10 +310,23 @@ mkdir -p ${EESSI_TMPDIR}
 [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
 # configure Singularity
-export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
-mkdir -p ${SINGULARITY_CACHEDIR}
+if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
+    export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
+    mkdir -p ${SINGULARITY_CACHEDIR}
+fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
+# pull & convert image and reset CONTAINER
+CONTAINER_URL_FMT=".*://(.*)"
+if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then
+    CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif
+    singularity pull ${CONTAINER_IMG} ${CONTAINER}
+    if [[ -x ${CONTAINER_IMG} ]]; then
+        CONTAINER="${PWD}/${CONTAINER_IMG}"
+    fi
+fi
+[[ ${INFO} -eq 1 ]] && echo "CONTAINER='${CONTAINER}'"
+
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}
 EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN}
@@ -326,8 +339,8 @@ mkdir -p ${EESSI_CVMFS_VAR_RUN}
 if [[ -z ${SINGULARITY_HOME} ]]; then
   export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
   mkdir -p ${EESSI_TMPDIR}/home
-  [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 fi
+[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}"
 
 # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined)
 BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs"

From 15ae58df240b1b1396611ee6fb5b5dbd50d4e793 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 20 Feb 2023 10:30:17 +0100
Subject: [PATCH 43/78] use VERBOSE instead of INFO

---
 .../workflows/test_eessi_container_script.yml | 22 +++++++++++++++++++
 eessi_container.sh                            |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 7678aedc64..5a44bae155 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -11,6 +11,8 @@ jobs:
       matrix:
         SCRIPT_TEST:
         - help
+        - listrepos_default
+        - listrepos_custom
         - run
         - shell
         - container
@@ -39,6 +41,26 @@ jobs:
               if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
                 ./eessi_container.sh --help
 
+              # test use of --list-repos without custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
+                outfile=out_listrepos.txt
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+              # test use of --list-repos with custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
+                outfile=out_listrepos.txt
+                outfile2=out_listrepos_2.txt
+                mkdir -p ${PWD}/cfg
+                echo "[EESSI/2021.12]" > cfg/repos.cfg
+                echo "[EESSI/2023.02]" >> cfg/repos.cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+                export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
+                grep "[EESSI/2023.02]" ${outfile2}
+
               # test use of --mode run
               elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
                 outfile=out_run.txt
diff --git a/eessi_container.sh b/eessi_container.sh
index cdd2c017fb..76a4d34143 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -325,7 +325,7 @@ if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then
         CONTAINER="${PWD}/${CONTAINER_IMG}"
     fi
 fi
-[[ ${INFO} -eq 1 ]] && echo "CONTAINER='${CONTAINER}'"
+[[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}"
 
 # set env vars and create directories for CernVM-FS
 EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB}

From 764e7138b0f8dfd82c81c1647bf78d04da1aec68 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 20 Feb 2023 10:48:38 +0100
Subject: [PATCH 44/78] fix test for --list-repos; improve output for
 --list-repos

---
 .../workflows/test_eessi_container_script.yml   |  2 ++
 eessi_container.sh                              | 17 ++++++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 5a44bae155..74cfd1fb10 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -53,7 +53,9 @@ jobs:
                 outfile2=out_listrepos_2.txt
                 mkdir -p ${PWD}/cfg
                 echo "[EESSI/2021.12]" > cfg/repos.cfg
+                echo "repo_version = 2021.12" >> cfg/repos.cfg
                 echo "[EESSI/2023.02]" >> cfg/repos.cfg
+                echo "repo_version = 2023.02" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
                 grep "EESSI-pilot" ${outfile}
 
diff --git a/eessi_container.sh b/eessi_container.sh
index 76a4d34143..dec082ce04 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -186,15 +186,18 @@ done
 
 set -- "${POSITIONAL_ARGS[@]}"
 
+
 if [[ ${LIST_REPOS} -eq 1 ]]; then
-    echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':"
+    echo "Listing available repositories with format 'name [source]':"
     echo "    EESSI-pilot [default]"
-    cfg_load ${EESSI_REPOS_CFG_FILE}
-    sections=$(cfg_sections)
-    while IFS= read -r repo_id
-    do
-        echo "    ${repo_id}"
-    done <<< "${sections}"
+    if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then
+        cfg_load ${EESSI_REPOS_CFG_FILE}
+        sections=$(cfg_sections)
+        while IFS= read -r repo_id
+        do
+            echo "    ${repo_id} [${EESSI_REPOS_CFG_FILE}]"
+        done <<< "${sections}"
+    fi
     exit 0
 fi
 

From 47cc1c6be44896675949567d88f5ae85a998b64f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 21 Feb 2023 21:27:56 +0100
Subject: [PATCH 45/78] implemented requested changes

---
 .../workflows/test_eessi_container_script.yml |  8 +--
 eessi_container.sh                            | 59 ++++++++++++++-----
 2 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index 74cfd1fb10..d4a9dfd53e 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -52,10 +52,10 @@ jobs:
                 outfile=out_listrepos.txt
                 outfile2=out_listrepos_2.txt
                 mkdir -p ${PWD}/cfg
-                echo "[EESSI/2021.12]" > cfg/repos.cfg
-                echo "repo_version = 2021.12" >> cfg/repos.cfg
-                echo "[EESSI/2023.02]" >> cfg/repos.cfg
-                echo "repo_version = 2023.02" >> cfg/repos.cfg
+                echo "[EESSI/20AB.CD]" > cfg/repos.cfg
+                echo "repo_version = 20AB.CD" >> cfg/repos.cfg
+                echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
+                echo "repo_version = 20HT.TP" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
                 grep "EESSI-pilot" ${outfile}
 
diff --git a/eessi_container.sh b/eessi_container.sh
index dec082ce04..86a510d56e 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -54,12 +54,11 @@ CVMFS_VAR_RUN="var-run-cvmfs"
 # directory for tmp used inside container
 export TMP_IN_CONTAINER=/tmp
 
-# repository cfg file, default name (default location: $PWD)
-#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
-export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg"
-# other repository cfg files in directory, default location: $PWD
-#   can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE
+# repository cfg directory and file
+#   directory: default $PWD or EESSI_REPOS_CFG_DIR_OVERRIDE if set
+#   file: directory + '/repos.cfg'
 export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}"
+export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg"
 
 
 # 0. parse args
@@ -288,7 +287,6 @@ else
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
   echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
 fi
-echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}"
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
 if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then
@@ -312,21 +310,52 @@ EESSI_TMPDIR=${EESSI_HOST_STORAGE}
 mkdir -p ${EESSI_TMPDIR}
 [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}"
 
-# configure Singularity
+# configure Singularity: if SINGULARITY_CACHEDIR is already defined, use that
+#   a global SINGULARITY_CACHEDIR would ensure that we don't consume
+#   storage space again and again for the container & also speed-up
+#   launch times across different sessions
 if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache
     mkdir -p ${SINGULARITY_CACHEDIR}
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
-# pull & convert image and reset CONTAINER
+# we try our best to make sure that we retain access to the container image in
+# a subsequent session ("best effort" only because pulling or copying operations
+# can fail ... in those cases the script may still succeed, but it is not
+# guaranteed that we have access to the same container when resuming later on)
+# - if CONTAINER references an image in a registry, pull & convert image
+#   and store it in ${EESSI_TMPDIR}
+#   + however, only pull image if there is no matching image in ${EESSI_TMPDIR} yet
+# - if CONTAINER references an image file, copy it to ${EESSI_TMPDIR}
+#   + however, only copy it if its base name does not yet exist in ${EESSI_TMPDIR}
+# - if the image file created (pulled or copied) or resumed exists in
+#   ${EESSI_TMPDIR}, let CONTAINER point to it
+#   + thus subsequent singularity commands in this script would just use the
+#     image file in EESSI_TMPDIR or the originally given source (some URL or
+#     path to an image file)
+CONTAINER_IMG=
 CONTAINER_URL_FMT=".*://(.*)"
-if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then
+if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
+    # replace : and - with _ in match (everything after ://) and append .sif
     CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif
-    singularity pull ${CONTAINER_IMG} ${CONTAINER}
-    if [[ -x ${CONTAINER_IMG} ]]; then
-        CONTAINER="${PWD}/${CONTAINER_IMG}"
+    # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
+    # resuming from a previous session)
+    if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+        singularity pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
     fi
+else
+    # determine file name as basename of CONTAINER
+    CONTAINER_IMG=$(basename ${CONTAINER})
+    # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when
+    # resuming from a previous session)
+    if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+        cp -a ${CONTAINER} ${EESSI_TMPDIR}/.
+    fi
+fi
+# let CONTAINER point to the pulled, copied or resumed image file
+if [[ -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+    CONTAINER="${EESSI_TMPDIR}/${CONTAINER_IMG}"
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}"
 
@@ -373,7 +402,7 @@ else
   cfg_load ${EESSI_REPOS_CFG_FILE}
 
   # copy repos.cfg to job directory --> makes it easier to inspect the job
-  cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
+  cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
 
   # cfg file should include: repo_name, repo_version, config_bundle,
   #   map { local_filepath -> container_filepath }
@@ -453,7 +482,8 @@ if [[ ! -z ${http_proxy} ]]; then
     [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'"
     echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \
        >> ${EESSI_TMPDIR}/repos_cfg/default.local
-    cat ${EESSI_TMPDIR}/repos_cfg/default.local
+    [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local"
+    [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local
 
     # if default.local is not BIND mounted into container, add it to BIND_PATHS
     if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then
@@ -537,7 +567,6 @@ if [[ ! -z ${SAVE} ]]; then
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
   echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
-  echo "RESUME_FROM_TGZ ${TGZ}"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From 712c40e567842da1c50b8c196fad0c9c5d30a69a Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 22:54:43 +0100
Subject: [PATCH 46/78] improvements to resuming job environment

cherry-picked via 23e773ce0d8c6504e1440629a4178318265b73d2
Note only applied part for bot/build.sh
---
 bot/build.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 8565511bfa..2c15b904f3 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -141,7 +141,7 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
-mkdir -p previous_tmp
+mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
 echo "./eessi_container.sh --access rw"
@@ -151,7 +151,7 @@ echo "                     ${HTTPS_PROXY_OPT}"
 echo "                     --info"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
-echo "                     --save ${PWD}/previous_tmp"
+echo "                     --save ${PWD}/previous_tmp/build_step"
 echo "                     --storage ${STORAGE}"
 echo "                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
@@ -163,7 +163,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      --info \
                      --mode run \
                      ${REPOSITORY_OPT} \
-                     --save ${PWD}/previous_tmp \
+                     --save ${PWD}/previous_tmp/build_step \
                      --storage ${STORAGE} \
                      ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
 
@@ -190,7 +190,7 @@ echo "                     --info"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --resume ${BUILD_TMPDIR}"
-echo "                     --save ${PWD}/previous_tmp"
+echo "                     --save ${PWD}/previous_tmp/tarball_step"
 echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
 ./eessi_container.sh --access rw \
                      ${CONTAINER_OPT} \
@@ -200,7 +200,7 @@ echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --resume ${BUILD_TMPDIR} \
-                     --save ${PWD}/previous_tmp \
+                     --save ${PWD}/previous_tmp/tarball_step \
                      ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 exit 0

From 16d98572d4148049a3a9f6a4258467eaf785d9f4 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:00:42 +0100
Subject: [PATCH 47/78] improved handling of container cache + explicit pull of
 image

cherry-picked via bfb1b29103dc309a12573f3ec9c247d432f08f73
Note, only applied part for bot/build.sh
---
 bot/build.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 2c15b904f3..06503e59a0 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -56,6 +56,12 @@ LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
 echo "LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
+SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE})
+echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    export SINGULARITY_CACHEDIR
+fi
+
 echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
 # replace any env variable in ${LOCAL_TMP} with its
 #   current value (e.g., a value that is local to the job)

From 9e8ca62cab90b013c6fb1b55180e37592aea02b5 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:02:35 +0100
Subject: [PATCH 48/78] changed --info -> --verbose

---
 bot/build.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 06503e59a0..77aa2d9077 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -154,7 +154,7 @@ echo "./eessi_container.sh --access rw"
 echo "                     ${CONTAINER_OPT}"
 echo "                     ${HTTP_PROXY_OPT}"
 echo "                     ${HTTPS_PROXY_OPT}"
-echo "                     --info"
+echo "                     --verbose"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --save ${PWD}/previous_tmp/build_step"
@@ -166,7 +166,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
                      ${HTTPS_PROXY_OPT} \
-                     --info \
+                     --verbose \
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --save ${PWD}/previous_tmp/build_step \
@@ -192,7 +192,7 @@ echo "./eessi_container.sh --access rw"
 echo "                     ${CONTAINER_OPT}"
 echo "                     ${HTTP_PROXY_OPT}"
 echo "                     ${HTTPS_PROXY_OPT}"
-echo "                     --info"
+echo "                     --verbose"
 echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --resume ${BUILD_TMPDIR}"
@@ -202,7 +202,7 @@ echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT
                      ${CONTAINER_OPT} \
                      ${HTTP_PROXY_OPT} \
                      ${HTTPS_PROXY_OPT} \
-                     --info \
+                     --verbose \
                      --mode run \
                      ${REPOSITORY_OPT} \
                      --resume ${BUILD_TMPDIR} \

From c0350a9da92f6632c9ba52a57a5a28e03cb9c040 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:14:38 +0100
Subject: [PATCH 49/78] improved messages for resume info and adjusted parsing
 in bot/build.sh

---
 bot/build.sh       | 2 +-
 eessi_container.sh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 77aa2d9077..f9d8c9cede 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -174,7 +174,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
-BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //")
+BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
 
 tar_outerr=$(mktemp tar.outerr.XXXX)
 timestamp=$(date +%s)
diff --git a/eessi_container.sh b/eessi_container.sh
index f1c755938c..060b003f33 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -284,7 +284,7 @@ else
     [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp"
   fi
   EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
-  echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)."
+  echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')."
 fi
 
 # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE}
@@ -565,7 +565,7 @@ if [[ ! -z ${SAVE} ]]; then
     TGZ=${SAVE}
   fi
   tar cf ${TGZ} -C ${EESSI_TMPDIR} .
-  echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')"
+  echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')"
 fi
 
 # TODO clean up tmp by default? only retain if another option provided (--retain-tmp)

From ef9a552aa1b0a96a7b13c4319b501e0a79459766 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 22 Feb 2023 23:25:27 +0100
Subject: [PATCH 50/78] delete one of two tmp storage tarballs created by
 bot/build.sh

---
 bot/build.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index f9d8c9cede..5926f005d5 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -209,4 +209,10 @@ echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT
                      --save ${PWD}/previous_tmp/tarball_step \
                      ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
+# if two tarballs have been generated, only keep the one from tarball step
+NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l)
+if [[ ${NUM_TARBALLS} -eq 2 ]]; then
+    rm -f previous_tmp/build_step/*.tgz
+fi
+
 exit 0

From b908b39bb64088141ed08b3af868dc759a298ff7 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Thu, 23 Feb 2023 23:00:19 +0100
Subject: [PATCH 51/78] update Lmod cache

---
 EESSI-pilot-install-software.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 8a646c07f5..9353e3ae84 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -483,6 +483,7 @@ if [ ! -f $LMOD_RC ]; then
     check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC"
 fi
 
+echo "Updating Lmod cache"
 $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
 
 $TOPDIR/check_missing_installations.sh

From 6974e4b80e3a59643d4eac0f229f50f8f547d430 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 13:33:54 +0100
Subject: [PATCH 52/78] add verbose messages on pulling/copying/reusing of
 container image

---
 eessi_container.sh | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 86a510d56e..01c69bdd92 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -320,6 +320,13 @@ if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
 fi
 [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}"
 
+# if VERBOSE is set to 0 (no arg --verbose), add argument '-q'
+if [[ ${VERBOSE} -eq 0 ]]; then
+    RUN_QUIET='-q'
+else
+    RUN_QUIET=''
+fi
+
 # we try our best to make sure that we retain access to the container image in
 # a subsequent session ("best effort" only because pulling or copying operations
 # can fail ... in those cases the script may still succeed, but it is not
@@ -342,7 +349,10 @@ if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
     # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
-        singularity pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
+        [[ ${VERBOSE} -eq 1 ]] && echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
+    else
+        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 else
     # determine file name as basename of CONTAINER
@@ -350,7 +360,10 @@ else
     # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
+        [[ ${VERBOSE} -eq 1 ]] && echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
         cp -a ${CONTAINER} ${EESSI_TMPDIR}/.
+    else
+        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 fi
 # let CONTAINER point to the pulled, copied or resumed image file
@@ -538,13 +551,6 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
-# if VERBOSE is set to 0 (no arg --verbose), add argument '-q'
-if [[ ${VERBOSE} -eq 0 ]]; then
-    RUN_QUIET='-q'
-else
-    RUN_QUIET=''
-fi
-
 echo "Launching container with command (next line):"
 echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@"
 singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@"

From 54bdea52e86d853a923d4884c4b62682440c5b09 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 13:37:47 +0100
Subject: [PATCH 53/78] don't require --verbose for info message on
 pulling/copying/reusing container image

---
 eessi_container.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index 01c69bdd92..e98cc53493 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -349,10 +349,10 @@ if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
     # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
-        [[ ${VERBOSE} -eq 1 ]] && echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
         singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER}
     else
-        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 else
     # determine file name as basename of CONTAINER
@@ -360,10 +360,10 @@ else
     # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then
-        [[ ${VERBOSE} -eq 1 ]] && echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}"
         cp -a ${CONTAINER} ${EESSI_TMPDIR}/.
     else
-        [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
+        echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}"
     fi
 fi
 # let CONTAINER point to the pulled, copied or resumed image file

From a4cea9afb12242340360c9ad922e13c8492c9d57 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 13:53:06 +0100
Subject: [PATCH 54/78] add check to make sure that container image exists in
 tmpdir

---
 .github/workflows/test_eessi_container_script.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml
index d4a9dfd53e..929fb22cec 100644
--- a/.github/workflows/test_eessi_container_script.yml
+++ b/.github/workflows/test_eessi_container_script.yml
@@ -108,6 +108,9 @@ jobs:
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 rm -f ${outfile}
 
+                # make sure that container image exists
+                test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1)
+
                 ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
                 cat ${outfile}
                 grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}

From c9758326512d824ae11e790f836c4db36dc3c527 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 24 Feb 2023 15:46:52 +0100
Subject: [PATCH 55/78] fix determining filename from container URL

---
 eessi_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index e98cc53493..1d76360735 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -344,8 +344,8 @@ fi
 CONTAINER_IMG=
 CONTAINER_URL_FMT=".*://(.*)"
 if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then
-    # replace : and - with _ in match (everything after ://) and append .sif
-    CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif
+    # replace ':', '-', '/' with '_' in match (everything after ://) and append .sif
+    CONTAINER_IMG="$(echo ${BASH_REMATCH[1]} | sed 's/[:\/-]/_/g').sif"
     # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when
     # resuming from a previous session)
     if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then

From 7cc65750c50bae0a2ea3f0cb9d96137fb21aa452 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 02:55:37 +0100
Subject: [PATCH 56/78] support cmd line arg flag terminator

---
 eessi_container.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/eessi_container.sh b/eessi_container.sh
index b1f8922d61..b9553a7ea1 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -173,6 +173,11 @@ while [[ $# -gt 0 ]]; do
       export https_proxy=${HTTPS_PROXY}
       shift 2
       ;;
+    --)
+      shift
+      POSITIONAL_ARGS+=("$@") # save positional args
+      break
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;

From 0def4e91ce9d52281e0e8fc3d30c96f095c75bd5 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 22:07:02 +0100
Subject: [PATCH 57/78] just print contents of cfg/job.cfg

---
 bot/build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bot/build.sh b/bot/build.sh
index 5926f005d5..906f1bfd7c 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -23,6 +23,10 @@
 # defaults
 export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
 
+echo "bot/build.sh: Showing job.cfg from software-layer side"
+cat cfg/job.cfg
+exit 0
+
 # source utils.sh
 source utils.sh
 

From ef608d15f20bea94bae116b7ece71a21586c7e3c Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 23:55:44 +0100
Subject: [PATCH 58/78] switching from json (YQ) to ini (cfg_files.sh)

---
 bot/build.sh | 43 ++++++++++++++++---------------------------
 1 file changed, 16 insertions(+), 27 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 906f1bfd7c..c621b9524f 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -17,50 +17,39 @@
 #  - the working directory contains a directory 'cfg' where the main config
 #    file 'job.cfg' has been deposited
 #  - the directory may contain any additional files references in job.cfg
-#  - the tool 'yq' for working with json files is available via the PATH or
-#    the environment variable BOT_YQ (see https://github.com/mikefarah/yq)
 
 # defaults
 export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
 
 echo "bot/build.sh: Showing job.cfg from software-layer side"
 cat cfg/job.cfg
-exit 0
-
-# source utils.sh
-source utils.sh
 
-# check setup / define key variables
-# get path for 'yq' (if not found, an empty string is returned)
-YQ=$(get_path_for_tool "yq" "BOT_YQ")
-exit_code=$?
-if [[ ${exit_code} -ne 0 ]]; then
-    fatal_error "could not find path to 'yq'; exiting"
-else
-    echo_green "found yq (${YQ})"
-fi
+# source utils.sh and cfg_files.sh
+source scripts/utils.sh
+source scripts/cfg_files.sh
 
 # check if './cfg/job.cfg' exists
 if [[ ! -r "${JOB_CFG_FILE}" ]]; then
     fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
 fi
 echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
+cfg_load ${JOB_CFG_FILE}
 
 # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
-HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE})
+HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
 HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
 echo "HTTP_PROXY='${HTTP_PROXY}'"
 
 # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
-HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE})
+HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
 HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
 echo "HTTPS_PROXY='${HTTPS_PROXY}'"
 
-LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
+LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
 echo "LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
-SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE})
+SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
 echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR
@@ -73,11 +62,11 @@ STORAGE=$(envsubst <<< ${LOCAL_TMP})
 echo "'${STORAGE}'"
 
 # obtain list of modules to be loaded
-LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE})
+LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
 echo "LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
-CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE})
+CONTAINER=$(cfg_get_value "repository" "container")
 export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
 export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
@@ -94,20 +83,20 @@ else
 fi
 
 # determine repository to be used from entry .repository in cfg/job.cfg
-REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
-EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE})
+REPOSITORY=$(cfg_get_value "repository" "repo_id")
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
 
 # determine pilot version to be used from .repository.repo_version in cfg/job.cfg
 # here, just set & export EESSI_PILOT_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
-export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE})
+export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
 
 # determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
 # "source init/eessi_defaults" via sourcing init/minimal_eessi_env
-export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE})
+export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
 
 
 # determine architecture to be used from entry .architecture in cfg/job.cfg
@@ -115,12 +104,12 @@ export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG
 if [[ ! -z "${CPU_TARGET}" ]]; then
     EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET}
 else
-    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
 fi
 export EESSI_SOFTWARE_SUBDIR_OVERRIDE
 
 # get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux)
-EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE})
+EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
 export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 
 # TODO

From 429d5a03ad0da75e8e8c52fb8897b341d1c70511 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Thu, 23 Feb 2023 23:56:28 +0100
Subject: [PATCH 59/78] add --generic arg when running install script

---
 bot/build.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index c621b9524f..387a9c95c9 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -140,6 +140,11 @@ REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
+GENERIC_OPT=
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then
+    GENERIC_OPT="--generic"
+fi
+
 mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
@@ -152,7 +157,7 @@ echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --save ${PWD}/previous_tmp/build_step"
 echo "                     --storage ${STORAGE}"
-echo "                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}"
+echo "                     ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
@@ -164,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ${REPOSITORY_OPT} \
                      --save ${PWD}/previous_tmp/build_step \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr}
+                     ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)

From bd31faa6b6e4e3945447118b6121667d01b4af7d Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 00:17:57 +0100
Subject: [PATCH 60/78] fix error in regex

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index 387a9c95c9..860f888a46 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -141,7 +141,7 @@ if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
 fi
 GENERIC_OPT=
-if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
     GENERIC_OPT="--generic"
 fi
 

From 1bafb7889fa8f49b183e048d63a8fbdabee041f1 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Fri, 24 Feb 2023 00:43:23 +0100
Subject: [PATCH 61/78] use command separator --

---
 bot/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 860f888a46..cdd59d6081 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -157,7 +157,7 @@ echo "                     --mode run"
 echo "                     ${REPOSITORY_OPT}"
 echo "                     --save ${PWD}/previous_tmp/build_step"
 echo "                     --storage ${STORAGE}"
-echo "                     ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
+echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
@@ -169,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                      ${REPOSITORY_OPT} \
                      --save ${PWD}/previous_tmp/build_step \
                      --storage ${STORAGE} \
-                     ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
+                     -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)

From c9041e34604ec881e6a5aeb96fbf31da91bf63a1 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 00:29:25 +0100
Subject: [PATCH 62/78] try fixing arg issue

---
 bot/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index cdd59d6081..f56f456dcf 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -161,7 +161,7 @@ echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh --access rw \
-                     ${CONTAINER_OPT} \
+                     "${CONTAINER_OPT}" \
                      ${HTTP_PROXY_OPT} \
                      ${HTTPS_PROXY_OPT} \
                      --verbose \

From 9d46a22892a4462f1b87c52c679d90cfed6256bb Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 00:41:36 +0100
Subject: [PATCH 63/78] put build args into an array

---
 bot/build.sh | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index f56f456dcf..8720c137a5 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -124,21 +124,30 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 #     files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg')
 
 # prepare options and directories for calling eessi_container.sh
+declare -a BUILD_STEP_ARGS=()
+BUILD_STEP_ARGS+=("--access" "rw")
+BUILD_STEP_ARGS+=("--mode" "run")
+BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step")
+BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
 CONTAINER_OPT=
 if [[ ! -z ${CONTAINER} ]]; then
     CONTAINER_OPT="--container ${CONTAINER}"
+    BUILD_STEP_ARGS+=("--container" "${CONTAINER}")
 fi
 HTTP_PROXY_OPT=
 if [[ ! -z ${HTTP_PROXY} ]]; then
     HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
+    BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
 fi
 HTTPS_PROXY_OPT=
 if [[ ! -z ${HTTPS_PROXY} ]]; then
     HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
+    BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
 fi
 REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
+    BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}")
 fi
 GENERIC_OPT=
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
@@ -148,27 +157,13 @@ fi
 mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
-echo "./eessi_container.sh --access rw"
-echo "                     ${CONTAINER_OPT}"
-echo "                     ${HTTP_PROXY_OPT}"
-echo "                     ${HTTPS_PROXY_OPT}"
+echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}"
 echo "                     --verbose"
-echo "                     --mode run"
-echo "                     ${REPOSITORY_OPT}"
-echo "                     --save ${PWD}/previous_tmp/build_step"
-echo "                     --storage ${STORAGE}"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
-./eessi_container.sh --access rw \
-                     "${CONTAINER_OPT}" \
-                     ${HTTP_PROXY_OPT} \
-                     ${HTTPS_PROXY_OPT} \
+./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \
                      --verbose \
-                     --mode run \
-                     ${REPOSITORY_OPT} \
-                     --save ${PWD}/previous_tmp/build_step \
-                     --storage ${STORAGE} \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from

From ff5045f0940730951c25cfb5260148655fbf933d Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 25 Feb 2023 01:19:25 +0100
Subject: [PATCH 64/78] use array to define args for build and tarball step

---
 bot/build.sh | 37 +++++++++++++++----------------------
 1 file changed, 15 insertions(+), 22 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 8720c137a5..bb1f4319a1 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -125,29 +125,39 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
 
 # prepare options and directories for calling eessi_container.sh
 declare -a BUILD_STEP_ARGS=()
+BUILD_STEP_ARGS+=("--verbose")
 BUILD_STEP_ARGS+=("--access" "rw")
 BUILD_STEP_ARGS+=("--mode" "run")
 BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
+declare -a TARBALL_STEP_ARGS=()
+TARBALL_STEP_ARGS+=("--verbose")
+TARBALL_STEP_ARGS+=("--access" "rw")
+TARBALL_STEP_ARGS+=("--mode" "run")
+TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step")
 CONTAINER_OPT=
 if [[ ! -z ${CONTAINER} ]]; then
     CONTAINER_OPT="--container ${CONTAINER}"
     BUILD_STEP_ARGS+=("--container" "${CONTAINER}")
+    TARBALL_STEP_ARGS+=("--container" "${CONTAINER}")
 fi
 HTTP_PROXY_OPT=
 if [[ ! -z ${HTTP_PROXY} ]]; then
     HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
     BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+    TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
 fi
 HTTPS_PROXY_OPT=
 if [[ ! -z ${HTTPS_PROXY} ]]; then
     HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
     BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+    TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
 fi
 REPOSITORY_OPT=
 if [[ ! -z ${REPOSITORY} ]]; then
     REPOSITORY_OPT="--repository ${REPOSITORY}"
     BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}")
+    TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}")
 fi
 GENERIC_OPT=
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
@@ -158,16 +168,15 @@ mkdir -p previous_tmp/{build_step,tarball_step}
 build_outerr=$(mktemp build.outerr.XXXX)
 echo "Executing command to build software:"
 echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}"
-echo "                     --verbose"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
 # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
 ./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \
-                     --verbose \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
+TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 
 tar_outerr=$(mktemp tar.outerr.XXXX)
 timestamp=$(date +%s)
@@ -181,26 +190,10 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $
 # /tmp as default?
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
-echo "./eessi_container.sh --access rw"
-echo "                     ${CONTAINER_OPT}"
-echo "                     ${HTTP_PROXY_OPT}"
-echo "                     ${HTTPS_PROXY_OPT}"
-echo "                     --verbose"
-echo "                     --mode run"
-echo "                     ${REPOSITORY_OPT}"
-echo "                     --resume ${BUILD_TMPDIR}"
-echo "                     --save ${PWD}/previous_tmp/tarball_step"
-echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
-./eessi_container.sh --access rw \
-                     ${CONTAINER_OPT} \
-                     ${HTTP_PROXY_OPT} \
-                     ${HTTPS_PROXY_OPT} \
-                     --verbose \
-                     --mode run \
-                     ${REPOSITORY_OPT} \
-                     --resume ${BUILD_TMPDIR} \
-                     --save ${PWD}/previous_tmp/tarball_step \
-                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}"
+echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \
+                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 # if two tarballs have been generated, only keep the one from tarball step
 NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l)

From d08bd211ed6c23c5b917b01bd01ef75a5cb7b1b9 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Tue, 7 Mar 2023 14:43:52 +0100
Subject: [PATCH 65/78] addressed comments and changes requested by first
 review

---
 bot/build.sh | 172 ++++++++++++++++++++++++---------------------------
 1 file changed, 82 insertions(+), 90 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index bb1f4319a1..004e2d4881 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -16,41 +16,44 @@
 #    pull request (OR by some other means)
 #  - the working directory contains a directory 'cfg' where the main config
 #    file 'job.cfg' has been deposited
-#  - the directory may contain any additional files references in job.cfg
+#  - the directory may contain any additional files referenced in job.cfg
 
-# defaults
-export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
-
-echo "bot/build.sh: Showing job.cfg from software-layer side"
-cat cfg/job.cfg
+# stop as soon as something fails
+set -e
 
 # source utils.sh and cfg_files.sh
 source scripts/utils.sh
 source scripts/cfg_files.sh
 
-# check if './cfg/job.cfg' exists
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# check if ${JOB_CFG_FILE} exists
 if [[ ! -r "${JOB_CFG_FILE}" ]]; then
     fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
 fi
-echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
+echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side"
+cat ${JOB_CFG_FILE}
+
+echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
 cfg_load ${JOB_CFG_FILE}
 
-# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
+# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy
 HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
 HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
-echo "HTTP_PROXY='${HTTP_PROXY}'"
+echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'"
 
-# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
+# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy
 HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
 HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
-echo "HTTPS_PROXY='${HTTPS_PROXY}'"
+echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'"
 
 LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
-echo "LOCAL_TMP='${LOCAL_TMP}'"
+echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
 SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
-echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
     export SINGULARITY_CACHEDIR
 fi
@@ -61,14 +64,21 @@ echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
 STORAGE=$(envsubst <<< ${LOCAL_TMP})
 echo "'${STORAGE}'"
 
+# make sure ${STORAGE} exists
+mkdir -p ${STORAGE}
+
+# make sure the base tmp storage is unique
+JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX)
+echo "bot/build.sh: created unique base tmp storage directory at ${JOB_STORAGE}"
+
 # obtain list of modules to be loaded
 LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
-echo "LOAD_MODULES='${LOAD_MODULES}'"
+echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
 CONTAINER=$(cfg_get_value "repository" "container")
-export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
-export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
+export SINGULARITY_HOME="${pwd}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
 # load modules if LOAD_MODULES is not empty
@@ -82,103 +92,91 @@ else
     echo "bot/build.sh: no modules to be loaded"
 fi
 
-# determine repository to be used from entry .repository in cfg/job.cfg
+# determine repository to be used from entry .repository in ${JOB_CFG_FILE}
 REPOSITORY=$(cfg_get_value "repository" "repo_id")
 EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
 
-# determine pilot version to be used from .repository.repo_version in cfg/job.cfg
+# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
 # here, just set & export EESSI_PILOT_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
 export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
 
-# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg
+# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
 # "source init/eessi_defaults" via sourcing init/minimal_eessi_env
 export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
-
-
-# determine architecture to be used from entry .architecture in cfg/job.cfg
-# default: leave empty to let downstream script(s) determine subdir to be used
-if [[ ! -z "${CPU_TARGET}" ]]; then
-    EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET}
-else
-    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
-fi
+echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
+
+# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE}
+# fallbacks:
+#  - ${CPU_TARGET} handed over from bot
+#  - left empty to let downstream script(s) determine subdir to be used
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}}
 export EESSI_SOFTWARE_SUBDIR_OVERRIDE
+echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'"
 
-# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux)
+# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux)
 EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
 export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
-
-# TODO
-#   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
-#     TODO ensure that the bot makes use of that. (currently sets env var
-#     CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also
-#     add it to cfg/job.cfg - .architecture.software_subdir)
-#   - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the
-#     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
-#     way to determine the IP address of a proxy)
-#   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
-#     files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg')
-
-# prepare options and directories for calling eessi_container.sh
+echo "bot/build.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
+
+# prepare arguments to eessi_container.sh common to build and tarball steps
+declare -a COMMON_ARGS=()
+COMMON_ARGS+=("--verbose")
+COMMON_ARGS+=("--access" "rw")
+COMMON_ARGS+=("--mode" "run")
+[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}")
+[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}")
+
+# make sure to use the same parent dir for storing tarballs of tmp
+PREVIOUS_TMP_DIR=${PWD}/previous_tmp
+
+# prepare directory to store tarball of tmp for build step
+TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step
+mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
+
+# prepare arguments to eessi_container.sh specific to build step
 declare -a BUILD_STEP_ARGS=()
-BUILD_STEP_ARGS+=("--verbose")
-BUILD_STEP_ARGS+=("--access" "rw")
-BUILD_STEP_ARGS+=("--mode" "run")
-BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step")
+BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
-declare -a TARBALL_STEP_ARGS=()
-TARBALL_STEP_ARGS+=("--verbose")
-TARBALL_STEP_ARGS+=("--access" "rw")
-TARBALL_STEP_ARGS+=("--mode" "run")
-TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step")
-CONTAINER_OPT=
-if [[ ! -z ${CONTAINER} ]]; then
-    CONTAINER_OPT="--container ${CONTAINER}"
-    BUILD_STEP_ARGS+=("--container" "${CONTAINER}")
-    TARBALL_STEP_ARGS+=("--container" "${CONTAINER}")
-fi
-HTTP_PROXY_OPT=
-if [[ ! -z ${HTTP_PROXY} ]]; then
-    HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
-    BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
-    TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}")
-fi
-HTTPS_PROXY_OPT=
-if [[ ! -z ${HTTPS_PROXY} ]]; then
-    HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
-    BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
-    TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
-fi
-REPOSITORY_OPT=
-if [[ ! -z ${REPOSITORY} ]]; then
-    REPOSITORY_OPT="--repository ${REPOSITORY}"
-    BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}")
-    TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}")
-fi
+
+# prepare arguments to install_software_layer.sh (specific to build step)
 GENERIC_OPT=
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
     GENERIC_OPT="--generic"
 fi
 
-mkdir -p previous_tmp/{build_step,tarball_step}
+# create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
+
 echo "Executing command to build software:"
-echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}"
 echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
-# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg
-export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
-./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \
+./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \
                      -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
 
+# prepare directory to store tarball of tmp for tarball step
+TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step
+mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR}
+
+# create tmp file for output of tarball step
+tar_outerr=$(mktemp tar.outerr.XXXX)
+
+# prepare arguments to eessi_container.sh specific to tarball step
+declare -a TARBALL_STEP_ARGS=()
+TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}")
+
 # determine temporary directory to resume from
 BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
 TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 
-tar_outerr=$(mktemp tar.outerr.XXXX)
 timestamp=$(date +%s)
 # to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now
 source init/eessi_defaults
@@ -190,15 +188,9 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $
 # /tmp as default?
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
-echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
 echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
-./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \
+./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
                      -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
-# if two tarballs have been generated, only keep the one from tarball step
-NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l)
-if [[ ${NUM_TARBALLS} -eq 2 ]]; then
-    rm -f previous_tmp/build_step/*.tgz
-fi
-
 exit 0

From f59bfdededf7b56bc95dc64f19021a3770d8ac61 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Tue, 7 Mar 2023 15:33:24 +0100
Subject: [PATCH 66/78] fix check for missing installations

---
 check_missing_installations.sh | 20 ++++++++++++++++++--
 scripts/utils.sh               |  2 +-
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index e4c75aebd8..45f276dee1 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -27,5 +27,21 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
 # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
 # PR #16531: Nextflow-22.10.1.eb
 ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}
-grep "No missing modules" ${eb_missing_out} > /dev/null
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+
+# the above assesses the installed software for each easyconfig provided in
+# the easystack file and then print messages such as
+# `No missing modules!`
+# or
+# `2 out of 3 required modules missing:`
+# depending on the result of the assessment. Hence, we need to check if the
+# output does not contain any line with ` required modules missing:`
+
+grep " required modules missing:" ${eb_missing_out} > /dev/null
+
+# we need to process the result (from finding `No missing modules` to NOT finding
+# ` required modules missing:` and no other error happened)
+#
+# if grep returns 1 (` required modules missing:` was NOT found), we set
+# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1
+[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1
+check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}"
diff --git a/scripts/utils.sh b/scripts/utils.sh
index d0da95e87f..1f46ae69b4 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -28,7 +28,7 @@ function check_exit_code {
     if [[ $ec -eq 0 ]]; then
         echo_green "${ok_msg}"
     else
-        fatal_error "${fail_msg}"
+        echo_red "${fail_msg}"
     fi
 }
 

From 7bb7c1de90b2bfb47a11124702d6dd968531267a Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:07:04 +0100
Subject: [PATCH 67/78] add test with missing package

---
 .github/workflows/test_eessi.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 92b1f71cad..1735bb77ed 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -40,4 +40,13 @@ jobs:
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
+              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
+              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
+              echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
+              tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml
+              ./check_missing_installations.sh
+              [[ $? -eq 0 ]] && exit 1 || exit 0

From 68a068b64ea5af8fad5a69ff812a5942cacd7167 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:25:04 +0100
Subject: [PATCH 68/78] separated test cases + updated evaluation of test
 result

---
 .github/workflows/test_eessi.yml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 1735bb77ed..073edaf4e8 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -42,11 +42,26 @@ jobs:
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
+
+        - name: Test check_missing_installations.sh with missing package
+          run: |
+              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              module load EasyBuild
+              eb --version
+              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
               tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml
-              ./check_missing_installations.sh
-              [[ $? -eq 0 ]] && exit 1 || exit 0
+              if ./check_missing_installations.sh; then
+                echo "captured missing package; test PASSED"
+                exit 0
+              else
+                echo "did NOT capture missing package; test FAILED"
+                exit 1
+              fi

From 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:30:17 +0100
Subject: [PATCH 69/78] fix small bug in env var name

---
 .github/workflows/test_eessi.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 073edaf4e8..f3bf2e7ab1 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -38,7 +38,7 @@ jobs:
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
@@ -50,7 +50,7 @@ jobs:
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml

From 094e82df3b0de5a195d99fa914674024aa2e8145 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:35:29 +0100
Subject: [PATCH 70/78] set software subdir override before sourcing init
 script

---
 .github/workflows/test_eessi.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index f3bf2e7ab1..d0b530e2e4 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -33,24 +33,24 @@ jobs:
 
         - name: Test check_missing_installations.sh script
           run: |
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
 
         - name: Test check_missing_installations.sh with missing package
           run: |
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml

From 60cf7e2715b7231810479a3fd91d20499c9b7670 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:42:01 +0100
Subject: [PATCH 71/78] Revert "set software subdir override before sourcing
 init script"

This reverts commit 094e82df3b0de5a195d99fa914674024aa2e8145.
---
 .github/workflows/test_eessi.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index d0b530e2e4..f3bf2e7ab1 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -33,24 +33,24 @@ jobs:
 
         - name: Test check_missing_installations.sh script
           run: |
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
 
         - name: Test check_missing_installations.sh with missing package
           run: |
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml

From 6f471520b63b0cee6b9d94818e7f42938dbc1bf8 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:42:51 +0100
Subject: [PATCH 72/78] Revert "fix small bug in env var name"

This reverts commit 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb.
---
 .github/workflows/test_eessi.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index f3bf2e7ab1..073edaf4e8 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -38,7 +38,7 @@ jobs:
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
@@ -50,7 +50,7 @@ jobs:
               eb --version
               export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
               echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml

From a15676ddac70b5cb18d82625a9a048d59199a41b Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 11:48:13 +0100
Subject: [PATCH 73/78] tweaking output of test

---
 .github/workflows/test_eessi.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 073edaf4e8..23b30c0063 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -43,7 +43,7 @@ jobs:
               echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
               ./check_missing_installations.sh
 
-        - name: Test check_missing_installations.sh with missing package
+        - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
           run: |
               source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
@@ -52,12 +52,12 @@ jobs:
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
-              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package"
+              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)"
               echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
-              tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml
+              tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
               if ./check_missing_installations.sh; then
                 echo "captured missing package; test PASSED"
                 exit 0

From 54d2a21f04f7e10ee8577c674be7b2d411da5026 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Wed, 8 Mar 2023 13:31:25 +0100
Subject: [PATCH 74/78] improved usage information

---
 eessi_container.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index b9553a7ea1..48c4653ba9 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -67,7 +67,7 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg"
 #    https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
 
 display_help() {
-  echo "usage: $0 [OPTIONS] [SCRIPT]"
+  echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]"
   echo " OPTIONS:"
   echo "  -a | --access {ro,rw}  - ro (read-only), rw (read & write) [default: ro]"
   echo "  -c | --container IMG   - image file or URL defining the container to use"
@@ -77,7 +77,7 @@ display_help() {
   echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
   echo "  -l | --list-repos      - list available repository identifiers [default: false]"
   echo "  -m | --mode MODE       - with MODE==shell (launch interactive shell) or"
-  echo "                           MODE==run (run a script) [default: shell]"
+  echo "                           MODE==run (run a script or command) [default: shell]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
   echo "                           repository to use [default: EESSI-pilot via"
   echo "                           default container, see --container]"
@@ -98,7 +98,9 @@ display_help() {
   echo "  -y | --https-proxy URL - provides URL for the env variable https_proxy"
   echo "                           [default: not set]; uses env var \$https_proxy if set"
   echo
-  echo " If value for --mode is 'run', the SCRIPT provided is executed."
+  echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If"
+  echo " arguments to the script/command start with '-' or '--', use the flag terminator"
+  echo " '--' to let eessi_container.sh stop parsing arguments."
 }
 
 # set defaults for command line arguments

From 99231e84eade092ae45fb5fa725d752eb04edf92 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 11 Mar 2023 00:07:09 +0100
Subject: [PATCH 75/78] if exit_code is not zero, use fatal_error (as it was)

---
 scripts/utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/utils.sh b/scripts/utils.sh
index 1f46ae69b4..d0da95e87f 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -28,7 +28,7 @@ function check_exit_code {
     if [[ $ec -eq 0 ]]; then
         echo_green "${ok_msg}"
     else
-        echo_red "${fail_msg}"
+        fatal_error "${fail_msg}"
     fi
 }
 

From b94bd111a3dcf422e7e78fa58bdbf52a74415fb2 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 11 Mar 2023 00:52:43 +0100
Subject: [PATCH 76/78] fix bug in test logic

---
 .github/workflows/test_eessi.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 23b30c0063..ce58fc3a97 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -59,9 +59,9 @@ jobs:
               echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
               tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
               if ./check_missing_installations.sh; then
-                echo "captured missing package; test PASSED"
-                exit 0
-              else
                 echo "did NOT capture missing package; test FAILED"
                 exit 1
+              else
+                echo "captured missing package; test PASSED"
+                exit 0
               fi

From 5e4bea9b71a37a09359def990fd12a1c01a5bf1c Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 11 Mar 2023 01:03:20 +0100
Subject: [PATCH 77/78] added explanation for test cases (exit code vs boolean
 logic)

---
 .github/workflows/test_eessi.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index ce58fc3a97..04195dd619 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -58,6 +58,11 @@ jobs:
               echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
               echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
               tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
+              # note, check_missing_installations.sh exits 1 if a package was
+              # missing, which is intepreted as false (exit code based, not
+              # boolean logic), hence when the script exits 0 if no package was
+              # missing it is interpreted as true, thus the test did not capture
+              # the missing package
               if ./check_missing_installations.sh; then
                 echo "did NOT capture missing package; test FAILED"
                 exit 1

From 04f28c8fc6e8e1956a10a5c4962f8d1117752e43 Mon Sep 17 00:00:00 2001
From: trz42 <trz42@users.noreply.github.com>
Date: Sat, 11 Mar 2023 01:33:50 +0100
Subject: [PATCH 78/78] fix bug, env var PWD is uppercase

---
 bot/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 004e2d4881..20334501ed 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -77,8 +77,8 @@ echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'"
 
 # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
 CONTAINER=$(cfg_get_value "repository" "container")
-export SINGULARITY_HOME="${pwd}:/eessi_bot_job"
-export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir"
+export SINGULARITY_HOME="${PWD}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir"
 mkdir -p ${SINGULARITY_TMPDIR}
 
 # load modules if LOAD_MODULES is not empty