diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 92b1f71cad..04195dd619 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -40,4 +40,33 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort + echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh + + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) + run: | + source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort + echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)" + echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml + tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml + # note, check_missing_installations.sh exits 1 if a package was + # missing, which is intepreted as false (exit code based, not + # boolean logic), hence when the script exits 0 if no package was + # missing it is interpreted as true, thus the test did not capture + # the missing package + if ./check_missing_installations.sh; then + echo "did NOT capture missing package; test FAILED" + exit 1 + else + echo "captured missing package; test PASSED" + exit 0 + fi diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml new file mode 100644 index 0000000000..929fb22cec --- /dev/null +++ b/.github/workflows/test_eessi_container_script.yml @@ -0,0 +1,135 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for eessi_container.sh script +on: [push, pull_request, workflow_dispatch] +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + eessi_container_script: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + SCRIPT_TEST: + - help + - listrepos_default + - listrepos_custom + - run + - shell + - container + - resume + # FIXME disabled because '--access rw' is not working in CI environment + #- readwrite + #- save + steps: + - name: Check out software-layer repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: install Apptainer + run: | + ./install_apptainer_ubuntu.sh + + - name: Collect info on test environment + run: | + mount + df -h + + - name: Test eessi_container.sh script + run: | + test_cmd="cat /etc/os-release" + out_pattern="Debian GNU/Linux 11" + + if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then + ./eessi_container.sh --help + + # test use of --list-repos without custom repos.cfg + elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then + outfile=out_listrepos.txt + ./eessi_container.sh --verbose --list-repos | tee ${outfile} + grep "EESSI-pilot" ${outfile} + + # test use of --list-repos with custom repos.cfg + elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then + outfile=out_listrepos.txt + outfile2=out_listrepos_2.txt + mkdir -p ${PWD}/cfg + echo "[EESSI/20AB.CD]" > cfg/repos.cfg + echo "repo_version = 20AB.CD" >> cfg/repos.cfg + echo "[EESSI/20HT.TP]" >> cfg/repos.cfg + echo "repo_version = 20HT.TP" >> cfg/repos.cfg + ./eessi_container.sh --verbose --list-repos | tee ${outfile} + grep "EESSI-pilot" ${outfile} + + export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg + ./eessi_container.sh --verbose --list-repos | tee ${outfile2} + grep "[EESSI/2023.02]" ${outfile2} + + # test use of --mode run + elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then + outfile=out_run.txt + echo "${test_cmd}" > test_script.sh + chmod u+x test_script.sh + export SINGULARITY_BIND="$PWD:/test" + ./eessi_container.sh --verbose --mode run /test/test_script.sh | tee ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of --mode shell + elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then + outfile=out_shell.txt + ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of --container option, using a totally different container; + # cfr. https://github.com/easybuilders/easybuild-containers + elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then + outfile=out_container.txt + container="docker://ghcr.io/eessi/build-node:debian10" + ./eessi_container.sh --verbose --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + grep "Debian GNU/Linux 10" ${outfile} + + # test use of '--access rw' to get write access in container + elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then + outfile=out_readwrite.txt + fn="test_${RANDOM}.txt" + echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh + chmod u+x test_script.sh + export SINGULARITY_BIND="$PWD:/test" + ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile} + + tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") + # note: must use '--access rw' again here, since touched file is in overlay upper dir + ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} + grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + + # test use of --resume + elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then + outfile=out_resume.txt + ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" > ${outfile} + + tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") + rm -f ${outfile} + + # make sure that container image exists + test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1) + + ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile} + cat ${outfile} + grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of --save (+ --resume) + elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then + outfile=out_save.txt + fn="test_${RANDOM}.txt" + test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" + ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} + rm -f ${outfile} + + ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} + grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + + tar tfv test-save.tar | grep "overlay-upper/${fn}" + + else + echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2 + exit 1 + fi diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 9c4975c381..94eb2894ee 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -6,8 +6,9 @@ on: - build_container.sh - install_software_layer.sh - run_in_compat_layer_env.sh - - utils.sh + - scripts/utils.sh - update_lmod_cache.sh + - create_directory_tarballs.sh pull_request: branches: @@ -16,8 +17,9 @@ on: - build_container.sh - install_software_layer.sh - run_in_compat_layer_env.sh - - utils.sh + - scripts/utils.sh - update_lmod_cache.sh + - create_directory_tarballs.sh permissions: contents: read # to fetch code (actions/checkout) jobs: @@ -27,18 +29,9 @@ jobs: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 - name: install Apptainer run: | - sudo apt-get install alien - alien --version - apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') - curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/$apptainer_rpm - sudo alien -d $apptainer_rpm - sudo apt install ./apptainer*.deb - apptainer --version - # also check whether 'singularity' command is still provided by Apptainer installation - singularity --version + ./install_apptainer_ubuntu.sh - name: test install_software_layer.sh script run: | @@ -49,3 +42,13 @@ jobs: # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh + + - name: test create_directory_tarballs.sh script + run: | + # scripts need to be copied to /tmp, + # since create_directory_tarballs.sh must be accessible from within build container + cp -a * /tmp/ + cd /tmp + ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12 + # check if tarballs have been produced + ls -l *.tar.gz diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 4a3c7b07b2..30c697d1e4 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -50,7 +50,7 @@ set -- "${POSITIONAL_ARGS[@]}" TOPDIR=$(dirname $(realpath $0)) -source $TOPDIR/utils.sh +source $TOPDIR/scripts/utils.sh # honor $TMPDIR if it is already defined, use /tmp otherwise if [ -z $TMPDIR ]; then @@ -484,6 +484,7 @@ if [ ! -f $LMOD_RC ]; then check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC" fi +echo "Updating Lmod cache" $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} $TOPDIR/check_missing_installations.sh diff --git a/bot/build.sh b/bot/build.sh index d5f81cd193..20334501ed 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -16,60 +16,69 @@ # pull request (OR by some other means) # - the working directory contains a directory 'cfg' where the main config # file 'job.cfg' has been deposited -# - the directory may contain any additional files references in job.cfg -# - the tool 'yq' for working with json files is available via the PATH or -# the environment variable BOT_YQ (see https://github.com/mikefarah/yq) +# - the directory may contain any additional files referenced in job.cfg -# defaults -export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +# stop as soon as something fails +set -e -# source utils.sh -source utils.sh +# source utils.sh and cfg_files.sh +source scripts/utils.sh +source scripts/cfg_files.sh -# check setup / define key variables -# get path for 'yq' (if not found, an empty string is returned) -YQ=$(get_path_for_tool "yq" "BOT_YQ") -exit_code=$? -if [[ ${exit_code} -ne 0 ]]; then - fatal_error "could not find path to 'yq'; exiting" -else - echo_green "found yq (${YQ})" -fi +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" -# check if './cfg/job.cfg' exists +# check if ${JOB_CFG_FILE} exists if [[ ! -r "${JOB_CFG_FILE}" ]]; then fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" fi -echo "obtaining configuration settings from '${JOB_CFG_FILE}'" +echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side" +cat ${JOB_CFG_FILE} + +echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" +cfg_load ${JOB_CFG_FILE} -# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy -HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE}) +# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy +HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} -echo "HTTP_PROXY='${HTTP_PROXY}'" +echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'" -# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy -HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE}) +# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy +HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} -echo "HTTPS_PROXY='${HTTPS_PROXY}'" +echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'" -LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) -echo "LOCAL_TMP='${LOCAL_TMP}'" +LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") +echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided +SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") +echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + export SINGULARITY_CACHEDIR +fi + echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " # replace any env variable in ${LOCAL_TMP} with its # current value (e.g., a value that is local to the job) STORAGE=$(envsubst <<< ${LOCAL_TMP}) echo "'${STORAGE}'" +# make sure ${STORAGE} exists +mkdir -p ${STORAGE} + +# make sure the base tmp storage is unique +JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX) +echo "bot/build.sh: created unique base tmp storage directory at ${JOB_STORAGE}" + # obtain list of modules to be loaded -LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE}) -echo "LOAD_MODULES='${LOAD_MODULES}'" +LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") +echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND -CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE}) -export SINGULARITY_HOME="$(pwd):/eessi_bot_job" -export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" +CONTAINER=$(cfg_get_value "repository" "container") +export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty @@ -83,99 +92,91 @@ else echo "bot/build.sh: no modules to be loaded" fi -# determine repository to be used from entry .repository in cfg/job.cfg -REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) -EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE}) +# determine repository to be used from entry .repository in ${JOB_CFG_FILE} +REPOSITORY=$(cfg_get_value "repository" "repo_id") +EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} +echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in cfg/job.cfg +# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} # here, just set & export EESSI_PILOT_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE}) +export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" -# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg +# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE}) - - -# determine architecture to be used from entry .architecture in cfg/job.cfg -# default: leave empty to let downstream script(s) determine subdir to be used -if [[ ! -z "${CPU_TARGET}" ]]; then - EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET} -else - EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) -fi +export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" + +# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} +# fallbacks: +# - ${CPU_TARGET} handed over from bot +# - left empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") +EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} export EESSI_SOFTWARE_SUBDIR_OVERRIDE +echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" -# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux) -EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE}) +# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) +EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} - -# TODO -# - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh -# TODO ensure that the bot makes use of that. (currently sets env var -# CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also -# add it to cfg/job.cfg - .architecture.software_subdir) -# - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the -# CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust -# way to determine the IP address of a proxy) -# - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying -# files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg') - -# prepare options and directories for calling eessi_container.sh -CONTAINER_OPT= -if [[ ! -z ${CONTAINER} ]]; then - CONTAINER_OPT="--container ${CONTAINER}" -fi -HTTP_PROXY_OPT= -if [[ ! -z ${HTTP_PROXY} ]]; then - HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" -fi -HTTPS_PROXY_OPT= -if [[ ! -z ${HTTPS_PROXY} ]]; then - HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" -fi -REPOSITORY_OPT= -if [[ ! -z ${REPOSITORY} ]]; then - REPOSITORY_OPT="--repository ${REPOSITORY}" -fi +echo "bot/build.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" + +# prepare arguments to eessi_container.sh common to build and tarball steps +declare -a COMMON_ARGS=() +COMMON_ARGS+=("--verbose") +COMMON_ARGS+=("--access" "rw") +COMMON_ARGS+=("--mode" "run") +[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") +[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") +[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") +[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") + +# make sure to use the same parent dir for storing tarballs of tmp +PREVIOUS_TMP_DIR=${PWD}/previous_tmp + +# prepare directory to store tarball of tmp for build step +TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step +mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} + +# prepare arguments to eessi_container.sh specific to build step +declare -a BUILD_STEP_ARGS=() +BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") +BUILD_STEP_ARGS+=("--storage" "${STORAGE}") + +# prepare arguments to install_software_layer.sh (specific to build step) GENERIC_OPT= if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then GENERIC_OPT="--generic" fi -mkdir -p previous_tmp/{build_step,tarball_step} +# create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) + echo "Executing command to build software:" -echo "./eessi_container.sh --access rw" -echo " ${CONTAINER_OPT}" -echo " ${HTTP_PROXY_OPT}" -echo " ${HTTPS_PROXY_OPT}" -echo " --info" -echo " --mode run" -echo " ${REPOSITORY_OPT}" -echo " --save ${PWD}/previous_tmp/build_step" -echo " --storage ${STORAGE}" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" -# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg -export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg -./eessi_container.sh --access rw \ - ${CONTAINER_OPT} \ - ${HTTP_PROXY_OPT} \ - ${HTTPS_PROXY_OPT} \ - --info \ - --mode run \ - ${REPOSITORY_OPT} \ - --save ${PWD}/previous_tmp/build_step \ - --storage ${STORAGE} \ +./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} -# determine temporary directory to resume from -BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //") +# prepare directory to store tarball of tmp for tarball step +TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step +mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR} +# create tmp file for output of tarball step tar_outerr=$(mktemp tar.outerr.XXXX) + +# prepare arguments to eessi_container.sh specific to tarball step +declare -a TARBALL_STEP_ARGS=() +TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}") + +# determine temporary directory to resume from +BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) +TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") + timestamp=$(date +%s) # to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now source init/eessi_defaults @@ -187,25 +188,9 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $ # /tmp as default? TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" -echo "./eessi_container.sh --access rw" -echo " ${CONTAINER_OPT}" -echo " ${HTTP_PROXY_OPT}" -echo " ${HTTPS_PROXY_OPT}" -echo " --info" -echo " --mode run" -echo " ${REPOSITORY_OPT}" -echo " --resume ${BUILD_TMPDIR}" -echo " --save ${PWD}/previous_tmp/tarball_step" -echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" -./eessi_container.sh --access rw \ - ${CONTAINER_OPT} \ - ${HTTP_PROXY_OPT} \ - ${HTTPS_PROXY_OPT} \ - --info \ - --mode run \ - ${REPOSITORY_OPT} \ - --resume ${BUILD_TMPDIR} \ - --save ${PWD}/previous_tmp/tarball_step \ - ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 30f9cc6ff7..45f276dee1 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -16,7 +16,7 @@ fi LOCAL_TMPDIR=$(mktemp -d) -source $TOPDIR/utils.sh +source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild @@ -27,5 +27,21 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used # PR #16531: Nextflow-22.10.1.eb ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} -grep "No missing modules" ${eb_missing_out} > /dev/null -check_exit_code $? "${ok_msg}" "${fail_msg}" + +# the above assesses the installed software for each easyconfig provided in +# the easystack file and then print messages such as +# `No missing modules!` +# or +# `2 out of 3 required modules missing:` +# depending on the result of the assessment. Hence, we need to check if the +# output does not contain any line with ` required modules missing:` + +grep " required modules missing:" ${eb_missing_out} > /dev/null + +# we need to process the result (from finding `No missing modules` to NOT finding +# ` required modules missing:` and no other error happened) +# +# if grep returns 1 (` required modules missing:` was NOT found), we set +# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 +[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}" diff --git a/create_init_tarball.sh b/create_directory_tarballs.sh similarity index 51% rename from create_init_tarball.sh rename to create_directory_tarballs.sh index 3393285a33..be282463eb 100755 --- a/create_init_tarball.sh +++ b/create_directory_tarballs.sh @@ -4,19 +4,6 @@ SOFTWARE_LAYER_TARBALL_URL=https://github.com/EESSI/software-layer/tarball/main set -eo pipefail -function echo_green() { - echo -e "\e[32m$1\e[0m" -} - -function echo_red() { - echo -e "\e[31m$1\e[0m" -} - -function error() { - echo_red "ERROR: $1" >&2 - exit 1 -} - if [ $# -ne 1 ]; then echo "Usage: $0 " >&2 exit 1 @@ -24,13 +11,18 @@ fi version=$1 +TOPDIR=$(dirname $(realpath $0)) + +source $TOPDIR/scripts/utils.sh + # Check if the EESSI version number encoded in the filename # is valid, i.e. matches the format YYYY.DD if ! echo "${version}" | egrep -q '^20[0-9][0-9]\.(0[0-9]|1[0-2])$' then - error "${version} is not a valid EESSI version." + fatal_error "${version} is not a valid EESSI version." fi +# Create tarball of init directory tartmp=$(mktemp -t -d init.XXXXX) mkdir "${tartmp}/${version}" tarname="eessi-${version}-init-$(date +%s).tar.gz" @@ -38,9 +30,20 @@ curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --s source "${tartmp}/${version}/init/minimal_eessi_env" if [ "${EESSI_PILOT_VERSION}" != "${version}" ] then - error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!" + fatal_error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!" fi tar czf "${tarname}" -C "${tartmp}" "${version}" rm -rf "${tartmp}" echo_green "Done! Created tarball ${tarname}." + +# Create tarball of scripts directory +# Version check has already been performed and would have caused script to exit at this point in case of problems +tartmp=$(mktemp -t -d scripts.XXXXX) +mkdir "${tartmp}/${version}" +tarname="eessi-${version}-scripts-$(date +%s).tar.gz" +curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards '*/scripts/' +tar czf "${tarname}" -C "${tartmp}" "${version}" +rm -rf "${tartmp}" + +echo_green "Done! Created tarball ${tarname}." diff --git a/eessi_container.sh b/eessi_container.sh index c9e4340f43..48c4653ba9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -12,6 +12,11 @@ # license: GPLv2 # +# -e: stop script as soon as any command has non-zero exit code +# -u: treat usage of undefined variables as errors +# FIXME commented out because it's OK (?) if some environment variables are not set (like $SINGULARITY_HOME) +# set -e -u + # script overview # -. initial settings & exit codes # 0. parse args @@ -23,13 +28,13 @@ # 6. save tmp (if requested) # -. initial settings & exit codes -base_dir=$(dirname $(realpath $0)) +TOPDIR=$(dirname $(realpath $0)) -source ${base_dir}/utils.sh -source ${base_dir}/cfg_files.sh +source ${TOPDIR}/scripts/utils.sh +source ${TOPDIR}/scripts/cfg_files.sh # exit codes: bitwise shift codes to allow for combination of exit codes -# ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh +# ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1)) ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) @@ -49,12 +54,11 @@ CVMFS_VAR_RUN="var-run-cvmfs" # directory for tmp used inside container export TMP_IN_CONTAINER=/tmp -# repository cfg file, default name (default location: $PWD) -# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg" -# other repository cfg files in directory, default location: $PWD -# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +# repository cfg directory and file +# directory: default $PWD or EESSI_REPOS_CFG_DIR_OVERRIDE if set +# file: directory + '/repos.cfg' export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}" +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" # 0. parse args @@ -63,20 +67,20 @@ export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}" # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash display_help() { - echo "usage: $0 [OPTIONS] [SCRIPT]" + echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -c | --container IMG - image file or URL defining the container to use" echo " [default: docker://ghcr.io/eessi/build-node:debian11]" echo " -h | --help - display this usage information [default: false]" - echo " -i | --info - display configuration information [default: false]" echo " -g | --storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" echo " -l | --list-repos - list available repository identifiers [default: false]" echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script) [default: shell]" + echo " MODE==run (run a script or command) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot]" + echo " repository to use [default: EESSI-pilot via" + echo " default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" echo " (check for output 'Using DIR as tmp ...' of a previous" @@ -88,23 +92,22 @@ display_help() { echo " when a directory is provided, the format of the" echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" echo " [default: not set]" + echo " -v | --verbose - display more information [default: false]" echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" echo " [default: not set]; uses env var \$http_proxy if set" echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" echo " [default: not set]; uses env var \$https_proxy if set" echo - echo " If value for --mode is 'run', the SCRIPT provided is executed." - echo - echo " FEATURES/OPTIONS to be implemented:" - echo " -d | --dry-run - run script except for executing the container," - echo " print information about setup [default: false]" + echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If" + echo " arguments to the script/command start with '-' or '--', use the flag terminator" + echo " '--' to let eessi_container.sh stop parsing arguments." } # set defaults for command line arguments ACCESS="ro" CONTAINER="docker://ghcr.io/eessi/build-node:debian11" -DRY_RUN=0 -INFO=0 +#DRY_RUN=0 +VERBOSE=0 STORAGE= LIST_REPOS=0 MODE="shell" @@ -114,8 +117,6 @@ SAVE= HTTP_PROXY=${http_proxy:-} HTTPS_PROXY=${https_proxy:-} -COMMAND_SEPARATOR=0 - POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do @@ -128,10 +129,10 @@ while [[ $# -gt 0 ]]; do CONTAINER="$2" shift 2 ;; - -d|--dry-run) - DRY_RUN=1 - shift 1 - ;; +# -d|--dry-run) +# DRY_RUN=1 +# shift 1 +# ;; -g|--storage) STORAGE="$2" shift 2 @@ -140,10 +141,6 @@ while [[ $# -gt 0 ]]; do display_help exit 0 ;; - -i|--info) - INFO=1 - shift 1 - ;; -l|--list-repos) LIST_REPOS=1 shift 1 @@ -164,6 +161,10 @@ while [[ $# -gt 0 ]]; do RESUME="$2" shift 2 ;; + -v|--verbose) + VERBOSE=1 + shift 1 + ;; -x|--http-proxy) HTTP_PROXY="$2" export http_proxy=${HTTP_PROXY} @@ -175,16 +176,12 @@ while [[ $# -gt 0 ]]; do shift 2 ;; --) - COMMAND_SEPARATOR=1 shift + POSITIONAL_ARGS+=("$@") # save positional args + break ;; -*|--*) - if [[ ${COMMAND_SEPARATOR} -eq 0 ]]; then - fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" - else - POSITIONAL_ARGS+=("$1") # save positional arg - shift - fi + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; *) # No more options POSITIONAL_ARGS+=("$1") # save positional arg @@ -196,14 +193,16 @@ done set -- "${POSITIONAL_ARGS[@]}" if [[ ${LIST_REPOS} -eq 1 ]]; then - echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':" + echo "Listing available repositories with format 'name [source]':" echo " EESSI-pilot [default]" - cfg_load ${EESSI_REPOS_CFG_FILE} - sections=$(cfg_sections) - while IFS= read -r repo_id - do - echo " ${repo_id}" - done <<< "${sections}" + if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + echo " ${repo_id} [${EESSI_REPOS_CFG_FILE}]" + done <<< "${sections}" + fi exit 0 fi @@ -289,12 +288,11 @@ else # mktemp falls back to using /tmp if TMPDIR is empty # TODO check if /tmp is writable, large enough and usable (different # features for ro-access and rw-access) - [[ ${INFO} -eq 1 ]] && echo "skipping sanity checks for /tmp" + [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) - echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." + echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')." fi -echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then @@ -316,41 +314,98 @@ fi # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} -[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR='${EESSI_TMPDIR}'" +[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" + +# configure Singularity: if SINGULARITY_CACHEDIR is already defined, use that +# a global SINGULARITY_CACHEDIR would ensure that we don't consume +# storage space again and again for the container & also speed-up +# launch times across different sessions +if [[ -z ${SINGULARITY_CACHEDIR} ]]; then + export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache + mkdir -p ${SINGULARITY_CACHEDIR} +fi +[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" -# configure Singularity -export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache -mkdir -p ${SINGULARITY_CACHEDIR} -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +# if VERBOSE is set to 0 (no arg --verbose), add argument '-q' +if [[ ${VERBOSE} -eq 0 ]]; then + RUN_QUIET='-q' +else + RUN_QUIET='' +fi + +# we try our best to make sure that we retain access to the container image in +# a subsequent session ("best effort" only because pulling or copying operations +# can fail ... in those cases the script may still succeed, but it is not +# guaranteed that we have access to the same container when resuming later on) +# - if CONTAINER references an image in a registry, pull & convert image +# and store it in ${EESSI_TMPDIR} +# + however, only pull image if there is no matching image in ${EESSI_TMPDIR} yet +# - if CONTAINER references an image file, copy it to ${EESSI_TMPDIR} +# + however, only copy it if its base name does not yet exist in ${EESSI_TMPDIR} +# - if the image file created (pulled or copied) or resumed exists in +# ${EESSI_TMPDIR}, let CONTAINER point to it +# + thus subsequent singularity commands in this script would just use the +# image file in EESSI_TMPDIR or the originally given source (some URL or +# path to an image file) +CONTAINER_IMG= +CONTAINER_URL_FMT=".*://(.*)" +if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then + # replace ':', '-', '/' with '_' in match (everything after ://) and append .sif + CONTAINER_IMG="$(echo ${BASH_REMATCH[1]} | sed 's/[:\/-]/_/g').sif" + # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when + # resuming from a previous session) + if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" + singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER} + else + echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" + fi +else + # determine file name as basename of CONTAINER + CONTAINER_IMG=$(basename ${CONTAINER}) + # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when + # resuming from a previous session) + if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" + cp -a ${CONTAINER} ${EESSI_TMPDIR}/. + else + echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" + fi +fi +# let CONTAINER point to the pulled, copied or resumed image file +if [[ -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + CONTAINER="${EESSI_TMPDIR}/${CONTAINER_IMG}" +fi +[[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}" # set env vars and create directories for CernVM-FS EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} mkdir -p ${EESSI_CVMFS_VAR_RUN} -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB='${EESSI_CVMFS_VAR_LIB}'" -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN='${EESSI_CVMFS_VAR_RUN}'" +[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" +[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" # allow that SINGULARITY_HOME is defined before script is run if [[ -z ${SINGULARITY_HOME} ]]; then export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" mkdir -p ${EESSI_TMPDIR}/home - [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'" fi +[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" # provide a '/tmp' inside the container BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" -[[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then # need to source defaults as late as possible (see other sourcing below) - source ${base_dir}/init/eessi_defaults + source ${TOPDIR}/init/eessi_defaults # strip "/cvmfs/" from default setting repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} @@ -366,7 +421,7 @@ else cfg_load ${EESSI_REPOS_CFG_FILE} # copy repos.cfg to job directory --> makes it easier to inspect the job - cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. + cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. # cfg file should include: repo_name, repo_version, config_bundle, # map { local_filepath -> container_filepath } @@ -396,7 +451,7 @@ else # convert config_map into associative array cfg_file_map cfg_init_file_map "${config_map}" - [[ ${INFO} -eq 1 ]] && cfg_print_map + [[ ${VERBOSE} -eq 1 ]] && cfg_print_map # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, # define BIND mounts and override repo name and version @@ -427,7 +482,7 @@ else export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" # need to source defaults as late as possible (after *_OVERRIDEs) - source ${base_dir}/init/eessi_defaults + source ${TOPDIR}/init/eessi_defaults fi # if http_proxy is not empty, we assume that the machine accesses internet @@ -437,16 +492,17 @@ fi if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is # http://SOME_HOSTNAME:SOME_PORT/ - [[ ${INFO} -eq 1 ]] && echo "http_proxy='${http_proxy}'" + [[ ${VERBOSE} -eq 1 ]] && echo "http_proxy='${http_proxy}'" PROXY_HOST=$(get_host_from_url ${http_proxy}) - [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" + [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" PROXY_PORT=$(get_port_from_url ${http_proxy}) - [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" + [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) - [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" + [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local - cat ${EESSI_TMPDIR}/repos_cfg/default.local + [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local" + [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local # if default.local is not BIND mounted into container, add it to BIND_PATHS if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then @@ -492,7 +548,7 @@ if [[ -z ${SINGULARITY_BIND} ]]; then else export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" fi -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" +[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" # pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then @@ -501,18 +557,10 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -# if INFO is set to 0 (no arg --info), add argument '-q' -if [[ ${INFO} -eq 0 ]]; then - RUN_QUIET='-q' -else - RUN_QUIET='' -fi - echo "Launching container with command (next line):" echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" -# TODO for now we run singularity with '-q' (quiet), later adjust this to the log level -# provided to the script singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" +exit_code=$? # 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then @@ -530,8 +578,10 @@ if [[ ! -z ${SAVE} ]]; then TGZ=${SAVE} fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . - echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" - echo "RESUME_FROM_TGZ ${TGZ}" + echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) + +# use exit code of container command +exit ${exit_code} diff --git a/init/eessi_defaults b/init/eessi_defaults index 5140e16742..f482cbc269 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -9,4 +9,4 @@ # export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" -export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" \ No newline at end of file +export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh new file mode 100755 index 0000000000..c35c34cda6 --- /dev/null +++ b/install_apptainer_ubuntu.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +# see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 +sudo apt-get install alien +alien --version +apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') +curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm} +sudo alien -d ${apptainer_rpm} +sudo apt install ./apptainer*.deb +apptainer --version +# also check whether 'singularity' command is still provided by Apptainer installation +singularity --version diff --git a/cfg_files.sh b/scripts/cfg_files.sh similarity index 100% rename from cfg_files.sh rename to scripts/cfg_files.sh diff --git a/utils.sh b/scripts/utils.sh similarity index 100% rename from utils.sh rename to scripts/utils.sh diff --git a/update_lmod_cache.sh b/update_lmod_cache.sh index 89e2ecbeee..814bb2dae0 100755 --- a/update_lmod_cache.sh +++ b/update_lmod_cache.sh @@ -5,7 +5,7 @@ TOPDIR=$(dirname $(realpath $0)) -source $TOPDIR/utils.sh +source $TOPDIR/scripts/utils.sh if [ $# -ne 2 ]; then echo "Usage: $0 " >&2