From bbc5d73e0d9b19b1c7a016ddb0c2da6790669c29 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 11:44:11 +0200 Subject: [PATCH 01/30] first version of bot/check-result.sh for software-layer - follows what has been implemented in the EESSI/eessi-bot-software-layer for checking the result of a job - re-uses code from a small script that was run manually on a bot instance to check a job's result - also re-uses code from the check-result.sh script in EESSI/compatibility-layer --- bot/check-result.sh | 195 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100755 bot/check-result.sh diff --git a/bot/check-result.sh b/bot/check-result.sh new file mode 100755 index 0000000000..38e76c3abc --- /dev/null +++ b/bot/check-result.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# +# Script to check the result of building the EESSI software layer. +# Intended use is that it is called by a (batch) job running on a compute +# node. +# +# This script is part of the EESSI compatibility layer, see +# https://github.com/EESSI/compatibility-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# result cases + +# - SUCCESS (all of) +# - working directory contains slurm-JOBID.out file +# - working directory contains eessi*tar.gz +# - no message ERROR +# - no message FAILED +# - no message ' required modules missing:' +# - one or more of 'No missing modules!' +# - message regarding created tarball +# - FAILED (one of ... implemented as NOT SUCCESS) +# - no slurm-JOBID.out file +# - no tarball +# - message with ERROR +# - message with FAILED +# - message with ' required modules missing:' +# - no message regarding created tarball + +# stop as soon as something fails +# set -e + +TOPDIR=$(dirname $(realpath $0)) + +source ${TOPDIR}/scripts/utils.sh +source ${TOPDIR}/scripts/cfg_files.sh + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " OPTIONS:" + echo " -h | --help - display this usage information [default: false]" + echo " -v | --verbose - display more information [default: false]" +} + +# set defaults for command line arguments +VERBOSE=0 + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + display_help + exit 0 + ;; + -v|--verbose) + VERBOSE=1 + shift 1 + ;; + --) + shift + POSITIONAL_ARGS+=("$@") # save positional args + break + ;; + -*|--*) + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +job_dir=${PWD} + +[[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" + +GP_slurm_out="slurm-${SLURM_JOB_ID}.out" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" +job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") +[[ $? -eq 0 ]] && SLURM=1 || SLURM=0 +[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" + +GP_error='ERROR: ' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" +grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && ERROR=1 || ERROR=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_failed='FAILED: ' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" +grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && FAILED=1 || FAILED=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_req_missing=' required modules missing:' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" +grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && MISSING=1 || MISSING=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_no_missing='No missing modules!' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" +grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_tgz_created="tar.gz created!" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" +grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) +TARBALL= +if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') +else + TGZ=0 +fi +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +echo "SUMMARY: ${job_dir}/${job_out}" +echo " test name : result (expected result)" +echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" + +if [[ ${SLURM} -eq 1 ]] && \ + [[ ${ERROR} -eq 0 ]] && \ + [[ ${FAILED} -eq 0 ]] && \ + [[ ${MISSING} -eq 0 ]] && \ + [[ ${NO_MISSING} -eq 1 ]] && \ + [[ ${TGZ} -eq 1 ]] && \ + [[ ! -z ${TARBALL} ]]; then + # SUCCESS + echo "[RESULT]" > ${job_result_file} + echo "summary = :grin: SUCCESS" >> ${job_result_file} + echo "details =" >> ${job_result_file} +else + # FAILURE + echo "[RESULT]" > ${job_result_file} + echo "summary = :cry: FAILURE" >> ${job_result_file} + echo "details =" >> ${job_result_file} +fi + +if [[ ${SLURM} -eq 1 ]]; then + # need to indent by 4 spaces + echo " job output file ${job_out} (pattern: ${GP_slurm_out})" >> ${job_result_file} +else + echo " no job output file matching ${GP_slurm_out}" >> ${job_result_file} +fi + +if [[ ${ERROR} -eq 0 ]]; then + echo " job output lacks message matching ${GP_error}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_error}" >> ${job_result_file} +fi + +if [[ ${FAILED} -eq 0 ]]; then + echo " job output lacks message matching ${GP_failed}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_failed}" >> ${job_result_file} +fi + +if [[ ${MISSING} -eq 0 ]]; then + echo " job output lacks message matching ${GP_req_missing}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_req_missing}" >> ${job_result_file} +fi + +if [[ ${NO_MISSING} -eq 1 ]]; then + echo " found message(s) matching ${GP_no_missing}" >> ${job_result_file} +else + echo " found no message matching ${GP_no_missing}" >> ${job_result_file} +fi + +if [[ ${TGZ} -eq 1 ]]; then + echo " found message matching ${GP_tgz_created}" >> ${job_result_file} +else + echo " found no message matching ${GP_tgz_created}" >> ${job_result_file} +fi + +echo "artefacts =" >> ${job_result_file} + +if [[ ! -z ${TARBALL} ]]; then + echo " ${TARBALL}" >> ${job_result_file} +fi + +exit 0 From fb04876141a464d98eb4cbb1aa3e34e4bc1a321c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 14:34:27 +0200 Subject: [PATCH 02/30] fix path to helper scripts + define result file name --- bot/check-result.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 38e76c3abc..15356f10cf 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -35,8 +35,8 @@ TOPDIR=$(dirname $(realpath $0)) -source ${TOPDIR}/scripts/utils.sh -source ${TOPDIR}/scripts/cfg_files.sh +source ${TOPDIR}/../scripts/utils.sh +source ${TOPDIR}/../scripts/cfg_files.sh display_help() { echo "usage: $0 [OPTIONS]" @@ -131,6 +131,8 @@ echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +job_result_file=_bot_job${SLURM_JOB_ID}.result + if [[ ${SLURM} -eq 1 ]] && \ [[ ${ERROR} -eq 0 ]] && \ [[ ${FAILED} -eq 0 ]] && \ From 5ebc5d92014b97738b85102d175edf77d14e2799 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 15:41:45 +0200 Subject: [PATCH 03/30] define functions for standardizing result output --- bot/check-result.sh | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 15356f10cf..dbc76e2dc9 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -151,41 +151,48 @@ else echo "details =" >> ${job_result_file} fi +function succeeded() { + echo "    :heavy_check_mark:${1}" +} + +function failed() { + echo "    :heavy_multiplication_x:${1}" +} + if [[ ${SLURM} -eq 1 ]]; then - # need to indent by 4 spaces - echo " job output file ${job_out} (pattern: ${GP_slurm_out})" >> ${job_result_file} + succeeded "job output file ${job_out}" >> ${job_result_file} else - echo " no job output file matching ${GP_slurm_out}" >> ${job_result_file} + failed "no job output file matching ${GP_slurm_out}" >> ${job_result_file} fi if [[ ${ERROR} -eq 0 ]]; then - echo " job output lacks message matching ${GP_error}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_error}" >> ${job_result_file} else - echo " job output contains message matching ${GP_error}" >> ${job_result_file} + failed "job output contains message matching ${GP_error}" >> ${job_result_file} fi if [[ ${FAILED} -eq 0 ]]; then - echo " job output lacks message matching ${GP_failed}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_failed}" >> ${job_result_file} else - echo " job output contains message matching ${GP_failed}" >> ${job_result_file} + failed "job output contains message matching ${GP_failed}" >> ${job_result_file} fi if [[ ${MISSING} -eq 0 ]]; then - echo " job output lacks message matching ${GP_req_missing}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_req_missing}" >> ${job_result_file} else - echo " job output contains message matching ${GP_req_missing}" >> ${job_result_file} + failed "job output contains message matching ${GP_req_missing}" >> ${job_result_file} fi if [[ ${NO_MISSING} -eq 1 ]]; then - echo " found message(s) matching ${GP_no_missing}" >> ${job_result_file} + succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} else - echo " found no message matching ${GP_no_missing}" >> ${job_result_file} + failed "found no message matching ${GP_no_missing}" >> ${job_result_file} fi if [[ ${TGZ} -eq 1 ]]; then - echo " found message matching ${GP_tgz_created}" >> ${job_result_file} + succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} else - echo " found no message matching ${GP_tgz_created}" >> ${job_result_file} + failed "found no message matching ${GP_tgz_created}" >> ${job_result_file} fi echo "artefacts =" >> ${job_result_file} From b8621545003d3d3e9bd482152a84f1aada2227ed Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 11:29:05 +0200 Subject: [PATCH 04/30] polishing formating of status messages --- bot/check-result.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index dbc76e2dc9..9e34f81dd8 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,11 +152,11 @@ else fi function succeeded() { - echo "    :heavy_check_mark:${1}" + echo " :heavy_check_mark:${1}" } function failed() { - echo "    :heavy_multiplication_x:${1}" + echo " :heavy_multiplication_x:${1}" } if [[ ${SLURM} -eq 1 ]]; then From 95e8ff8e55f316852aeb1ec09ce94db5cacb1b62 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 12:29:56 +0200 Subject: [PATCH 05/30] more polishing of comment details --- bot/check-result.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 9e34f81dd8..2d2f33be06 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -166,33 +166,33 @@ else fi if [[ ${ERROR} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_error}" >> ${job_result_file} + succeeded "no message matching ${GP_error}" >> ${job_result_file} else - failed "job output contains message matching ${GP_error}" >> ${job_result_file} + failed "found message matching ${GP_error}" >> ${job_result_file} fi if [[ ${FAILED} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_failed}" >> ${job_result_file} + succeeded "no message matching ${GP_failed}" >> ${job_result_file} else - failed "job output contains message matching ${GP_failed}" >> ${job_result_file} + failed "found message matching ${GP_failed}" >> ${job_result_file} fi if [[ ${MISSING} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_req_missing}" >> ${job_result_file} + succeeded "no message matching ${GP_req_missing}" >> ${job_result_file} else - failed "job output contains message matching ${GP_req_missing}" >> ${job_result_file} + failed "found message matching ${GP_req_missing}" >> ${job_result_file} fi if [[ ${NO_MISSING} -eq 1 ]]; then succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} else - failed "found no message matching ${GP_no_missing}" >> ${job_result_file} + failed "no message matching ${GP_no_missing}" >> ${job_result_file} fi if [[ ${TGZ} -eq 1 ]]; then succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} else - failed "found no message matching ${GP_tgz_created}" >> ${job_result_file} + failed "no message matching ${GP_tgz_created}" >> ${job_result_file} fi echo "artefacts =" >> ${job_result_file} From 6724b49b18dc0a4cabc87c5526e0e8741b148686 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 13:06:15 +0200 Subject: [PATCH 06/30] minor polishing of job result messages --- bot/check-result.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 2d2f33be06..1598e9ca1b 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,11 +152,11 @@ else fi function succeeded() { - echo " :heavy_check_mark:${1}" + echo " :heavy_check_mark: ${1}" } function failed() { - echo " :heavy_multiplication_x:${1}" + echo " :heavy_multiplication_x: ${1}" } if [[ ${SLURM} -eq 1 ]]; then From b70c858d65eadba24366518d30f63a3fdca3b686 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 13:36:15 +0200 Subject: [PATCH 07/30] only log if run with --verbose --- bot/check-result.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 1598e9ca1b..55276c405a 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -123,13 +123,13 @@ else fi [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" -echo "SUMMARY: ${job_dir}/${job_out}" -echo " test name : result (expected result)" -echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" -echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" +[[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" +[[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" job_result_file=_bot_job${SLURM_JOB_ID}.result From 677c36296f1ab433964b8caa230ad8ef32c5d625 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 20 Apr 2023 15:22:40 +0200 Subject: [PATCH 08/30] fix small bug when checking for tarball msg --- bot/check-result.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 55276c405a..3f0c89ea21 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -112,9 +112,9 @@ grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_tgz_created="tar.gz created!" +TARBALL= [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) -TARBALL= if [[ $? -eq 0 ]]; then TGZ=1 TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') From e4932eefaff86adee6e91265efdfd5268efdf52a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 20 Apr 2023 15:31:36 +0200 Subject: [PATCH 09/30] avoid creating false positives --- bot/check-result.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 3f0c89ea21..e28ae4e38e 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -82,38 +82,42 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" GP_slurm_out="slurm-${SLURM_JOB_ID}.out" -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") [[ $? -eq 0 ]] && SLURM=1 || SLURM=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" GP_error='ERROR: ' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_failed='FAILED: ' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_req_missing=' required modules missing:' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_no_missing='No missing modules!' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_tgz_created="tar.gz created!" TARBALL= -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) if [[ $? -eq 0 ]]; then TGZ=1 @@ -121,6 +125,8 @@ if [[ $? -eq 0 ]]; then else TGZ=0 fi +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" From 9a0e220db5a21b6285785ab09683bc0bb62e3108 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Jun 2023 18:48:43 +0200 Subject: [PATCH 10/30] also filter OpenSSL dependency in EasyBuild configuration --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index 245553f342..5d4e9b5a48 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -26,7 +26,7 @@ fi # note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49) # filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104) # problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency) -DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib +DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,OpenSSL,util-linux,XZ,zlib # For aarch64 we need to also filter out Yasm. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190 if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then From 0a41c207e0ee1646cff53768cc86f83a8f740205 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 08:14:02 +0200 Subject: [PATCH 11/30] bump version to 2023.06 --- init/eessi_defaults | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/eessi_defaults b/init/eessi_defaults index f482cbc269..0143dc38ab 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -9,4 +9,4 @@ # export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" -export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" +export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2023.06}" From 88a758a4c250111d35ca3c76e1516bd62c3e2620 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 08:14:45 +0200 Subject: [PATCH 12/30] configure EasyBuild to allow experimental features (like using easystack files) --- configure_easybuild | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure_easybuild b/configure_easybuild index 245553f342..23f3920154 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -36,3 +36,6 @@ fi export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER export EASYBUILD_MODULE_EXTENSIONS=1 + +# need to enable use of experimental features, since we're using easystack files +export EASYBUILD_EXPERIMENTAL=1 From 8aa1c25b69e84a76ce286029bb320e910e8aff3b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 08:23:22 +0200 Subject: [PATCH 13/30] update install script to use EasyBuild 4.7.2 + use easystack file for foss/2021a --- EESSI-pilot-install-software.sh | 244 +++---------------------------- eessi-2023.04-eb-4.7.2-2021a.yml | 6 + 2 files changed, 24 insertions(+), 226 deletions(-) create mode 100644 eessi-2023.04-eb-4.7.2-2021a.yml diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index d9bcf20231..f58903f8cd 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -137,235 +137,27 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -REQ_EB_VERSION='4.5.0' - -# load EasyBuild module (will be installed if it's not available yet) -source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION} - -echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." - -# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, -# see https://github.com/EESSI/software-layer/issues/123 -# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 -ok_msg="Java installed, off to a good (?) start!" -fail_msg="Failed to install Java, woopsie..." -$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install GCC for foss/2020a -export GCC_EC="GCC-9.3.0.eb" -echo ">> Starting slow with ${GCC_EC}..." -ok_msg="${GCC_EC} installed, yippy! Off to a good start..." -fail_msg="Installation of ${GCC_EC} failed!" -# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, -# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place -$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install CMake with custom easyblock that patches CMake when --sysroot is used -echo ">> Install CMake with fixed easyblock to take into account --sysroot" -ok_msg="CMake installed!" -fail_msg="Installation of CMake failed, what the ..." -$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 -echo ">> Installing OpenBLAS..." -ok_msg="Done with OpenBLAS!" -fail_msg="Installation of OpenBLAS failed!" -if [[ $GENERIC -eq 1 ]]; then - echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" -else - openblas_include_easyblocks_from_pr='' -fi -$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OpenMPI..." -ok_msg="OpenMPI installed, w00!" -fail_msg="Installation of OpenMPI failed, that's not good..." -$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install Python -echo ">> Install Python 2.7.18 and Python 3.8.2..." -ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" -fail_msg="Installation of Python failed, oh no..." -$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Perl..." -ok_msg="Perl installed, making progress..." -fail_msg="Installation of Perl failed, this never happens..." -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Qt5..." -ok_msg="Qt5 installed, phieuw, that was a big one!" -fail_msg="Installation of Qt5 failed, that's frustrating..." -$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# skip test step when installing SciPy-bundle on aarch64, -# to dance around problem with broken numpy tests; -# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 -echo ">> Installing SciPy-bundle" -ok_msg="SciPy-bundle installed, yihaa!" -fail_msg="SciPy-bundle installation failed, bummer..." -SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb -if [[ "$(uname -m)" == "aarch64" ]]; then - $EB $SCIPY_EC --robot --skip-test-step -else - $EB $SCIPY_EC --robot -fi -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing GROMACS..." -ok_msg="GROMACS installed, wow!" -fail_msg="Installation of GROMACS failed, damned..." -$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! -# 32GB is sufficient to build with 16 cores -echo ">> Installing OpenFOAM (twice!)..." -ok_msg="OpenFOAM installed, now we're talking!" -fail_msg="Installation of OpenFOAM failed, we were so close..." -$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - echo ">> Installing QuantumESPRESSO..." - ok_msg="QuantumESPRESSO installed, let's go quantum!" - fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" - $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi +for eb_version in '4.7.2'; do -echo ">> Installing R 4.0.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Bioconductor 3.11 bundle..." -ok_msg="Bioconductor installed, enjoy!" -fail_msg="Installation of Bioconductor failed, that's annoying..." -$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing TensorFlow 2.3.1..." -ok_msg="TensorFlow 2.3.1 installed, w00!" -fail_msg="Installation of TensorFlow failed, why am I not surprised..." -$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Horovod 0.21.3..." -ok_msg="Horovod installed! Go do some parallel training!" -fail_msg="Horovod installation failed. There comes the headache..." -$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - - echo ">> Installing code-server 3.7.3..." - ok_msg="code-server 3.7.3 installed, now you can use VS Code!" - fail_msg="Installation of code-server failed, that's going to be hard to fix..." - $EB code-server-3.7.3.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi + # load EasyBuild module (will be installed if it's not available yet) + source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION} -echo ">> Installing RStudio-Server 1.3.1093..." -ok_msg="RStudio-Server installed, enjoy!" -fail_msg="Installation of RStudio-Server failed, might be OS deps..." -$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Spark 3.1.1..." -ok_msg="Spark installed, set off the fireworks!" -fail_msg="Installation of Spark failed, no fireworks this time..." -$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing IPython 7.15.0..." -ok_msg="IPython installed, launch your Jupyter Notebooks!" -fail_msg="Installation of IPython failed, that's unexpected..." -$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing WRF 3.9.1.1..." -ok_msg="WRF installed, it's getting hot in here!" -fail_msg="Installation of WRF failed, that's unexpected..." -OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing R 4.1.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Nextflow 22.10.1..." -ok_msg="Nextflow installed, the work must flow..." -fail_msg="Installation of Nextflow failed, that's unexpected..." -$EB -r --from-pr 16531 Nextflow-22.10.1.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing EasyBuild 4.5.1..." -ok_msg="EasyBuild v4.5.1 installed" -fail_msg="EasyBuild v4.5.1 failed to install" -$EB --from-pr 14545 --include-easyblocks-from-pr 2805 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 -check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" - -echo ">> Installing SciPy-bundle with foss/2021a..." -ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" -fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." -# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 -# which includes patch to fix installation with recent Linux kernel headers -$EB --from-pr 14454 GCCcore-10.3.0.eb --robot -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 -# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set -# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 -# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 -# that includes patch to fix bootstrap problem when using alternate sysroot -$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot -# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 -# which includes a patch to fix installation on POWER -$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot -# ignore failing FlexiBLAS tests when building on POWER; -# some tests are failing due to a segmentation fault due to "invalid memory reference", -# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; -# using -fstack-protector-strong -fstack-clash-protection should fix that, -# but it doesn't for some reason when building for ppc64le/generic... -if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure -else - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb -fi + echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." + + for gen in '2021a'; do + + es="eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-${gen}.yml" -$EB SciPy-bundle-2021.05-foss-2021a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" + if [ -f ${es} ]; then + echo_green "Feeding easystack file ${es} to EasyBuild..." + + ${EB} --easystack ${TOPDIR}/${es} -M && ${EB} --easystack ${TOPDIR}/${es} --robot + else + fatal_error "Easystack file ${es} not found!" + fi + done + +done ### add packages here diff --git a/eessi-2023.04-eb-4.7.2-2021a.yml b/eessi-2023.04-eb-4.7.2-2021a.yml new file mode 100644 index 0000000000..9b1312ba9a --- /dev/null +++ b/eessi-2023.04-eb-4.7.2-2021a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - GCC-10.3.0 + - CMake-3.20.1-GCCcore-10.3.0.eb: + options: + include-easyblocks-from-pr: 2248 + - foss-2021a From b7870afd2e543cd19a07b4cafb94a9e1633b71b0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 08:24:00 +0200 Subject: [PATCH 14/30] implement pre-configure hook for OpenBLAS to use DYNAMIC_ARCH=1 when building for a generic CPU target --- eb_hooks.py | 56 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 2fba925b01..787e990e75 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -11,6 +11,13 @@ from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC +# prefer importing LooseVersion from easybuild.tools, but fall back to distuils in case EasyBuild <= 4.7.0 is used +try: + from easybuild.tools import LooseVersion +except ImportError: + from distutils.version import LooseVersion + + EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -56,13 +63,6 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) -def pre_configure_hook(self, *args, **kwargs): - """Main pre-configure hook: trigger custom functions based on software name.""" - - if self.name in PRE_CONFIGURE_HOOKS: - PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) - - def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" @@ -91,7 +91,7 @@ def pre_prepare_hook(self, *args, **kwargs): mpi_family, rpath_override_dirs) -def gcc_postprepare(self, *args, **kwargs): +def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs): """ Post-configure hook for GCCcore: - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' @@ -121,6 +121,7 @@ def gcc_postprepare(self, *args, **kwargs): else: raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!") + def post_prepare_hook(self, *args, **kwargs): """Main post-prepare hook: trigger custom functions.""" @@ -134,7 +135,7 @@ def post_prepare_hook(self, *args, **kwargs): POST_PREPARE_HOOKS[self.name](self, *args, **kwargs) -def cgal_toolchainopts_precise(ec, eprefix): +def parse_hook_cgal_toolchainopts_precise(ec, eprefix): """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER.""" if ec.name == 'CGAL': if get_cpu_architecture() == POWER: @@ -147,7 +148,7 @@ def cgal_toolchainopts_precise(ec, eprefix): raise EasyBuildError("CGAL-specific hook triggered for non-CGAL easyconfig?!") -def fontconfig_add_fonts(ec, eprefix): +def parse_hook_fontconfig_add_fonts(ec, eprefix): """Inject --with-add-fonts configure option for fontconfig.""" if ec.name == 'fontconfig': # make fontconfig aware of fonts included with compat layer @@ -158,7 +159,7 @@ def fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") -def ucx_eprefix(ec, eprefix): +def parse_hook_ucx_eprefix(ec, eprefix): """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': ec.update('configopts', '--with-sysroot=%s' % eprefix) @@ -174,7 +175,19 @@ def pre_configure_hook(self, *args, **kwargs): PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) -def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): +def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): + """ + Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC + """ + if self.name == 'OpenBLAS': + if build_option('optarch') == OPTARCH_GENERIC: + for step in ('build', 'test', 'install'): + self.cfg.update(f'{step}opts', "DYNAMIC_ARCH=1") + else: + raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") + + +def pre_configure_hook_libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): """Add --disable-psm3 to libfabric configure options when building with --optarch=GENERIC on x86_64.""" if self.name == 'libfabric': if get_cpu_architecture() == X86_64: @@ -187,7 +200,7 @@ def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): raise EasyBuildError("libfabric-specific hook triggered for non-libfabric easyconfig?!") -def metabat_preconfigure(self, *args, **kwargs): +def pre_configure_hook_metabat_filtered_zlib_dep(self, *args, **kwargs): """ Pre-configure hook for MetaBAT: - take into account that zlib is a filtered dependency, @@ -201,7 +214,7 @@ def metabat_preconfigure(self, *args, **kwargs): raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!") -def wrf_preconfigure(self, *args, **kwargs): +def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): """ Pre-configure hook for WRF: - patch arch/configure_new.defaults so building WRF with foss toolchain works on aarch64 @@ -217,17 +230,18 @@ def wrf_preconfigure(self, *args, **kwargs): PARSE_HOOKS = { - 'CGAL': cgal_toolchainopts_precise, - 'fontconfig': fontconfig_add_fonts, - 'UCX': ucx_eprefix, + 'CGAL': parse_hook_cgal_toolchainopts_precise, + 'fontconfig': parse_hook_fontconfig_add_fonts, + 'UCX': parse_hook_ucx_eprefix, } POST_PREPARE_HOOKS = { - 'GCCcore': gcc_postprepare, + 'GCCcore': post_prepare_hook_gcc_prefixed_ld_rpath_wrapper, } PRE_CONFIGURE_HOOKS = { - 'libfabric': libfabric_disable_psm3_x86_64_generic, - 'MetaBAT': metabat_preconfigure, - 'WRF': wrf_preconfigure, + 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, + 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, + 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, + 'WRF': pre_configure_hook_wrf_aarch64, } From a4790b442d2f44b84f3be1bb1f025ed44247586c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 17:20:42 +0200 Subject: [PATCH 15/30] consistently use ${eb_version} rather than ${REQ_EB_VERSION} --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index f58903f8cd..74887db93b 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -140,7 +140,7 @@ fi for eb_version in '4.7.2'; do # load EasyBuild module (will be installed if it's not available yet) - source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION} + source ${TOPDIR}/load_easybuild_module.sh ${eb_version} echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." From 9cdc38e34f6b00ce49b2ac1b24b82c473e028433 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 17:24:09 +0200 Subject: [PATCH 16/30] rename easystack file for EESSI 2023.06 to use correct EESSI version --- ...2023.04-eb-4.7.2-2021a.yml => eessi-2023.06-eb-4.7.2-2021a.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename eessi-2023.04-eb-4.7.2-2021a.yml => eessi-2023.06-eb-4.7.2-2021a.yml (100%) diff --git a/eessi-2023.04-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml similarity index 100% rename from eessi-2023.04-eb-4.7.2-2021a.yml rename to eessi-2023.06-eb-4.7.2-2021a.yml From 64132649925ce3f040a4551589eeef8f9df9c0c3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 18:57:54 +0200 Subject: [PATCH 17/30] update check_missing_installations.sh to take path to easystack file as an argument, and hoist it into for loop that iterates over all easystack files --- EESSI-pilot-install-software.sh | 6 +++--- check_missing_installations.sh | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 74887db93b..f2d50dfe91 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -151,7 +151,9 @@ for eb_version in '4.7.2'; do if [ -f ${es} ]; then echo_green "Feeding easystack file ${es} to EasyBuild..." - ${EB} --easystack ${TOPDIR}/${es} -M && ${EB} --easystack ${TOPDIR}/${es} --robot + ${EB} --easystack ${TOPDIR}/${es} --missing && ${EB} --easystack ${TOPDIR}/${es} --robot + + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es} else fatal_error "Easystack file ${es} not found!" fi @@ -170,7 +172,5 @@ fi $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} -$TOPDIR/check_missing_installations.sh - echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 4a5316c09f..3627d1d0b5 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to check for missing installations in EESSI pilot software stack (version 2021.12) +# Script to check for missing installations in EESSI pilot software stack (version 2023.06) # # author: Kenneth Hoste (@boegel) # author: Thomas Roeblitz (@trz42) @@ -10,10 +10,11 @@ TOPDIR=$(dirname $(realpath $0)) -if [ -z ${EESSI_PILOT_VERSION} ]; then - echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2 +if [ $# -ne 1 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi +easystack=$1 LOCAL_TMPDIR=$(mktemp -d) @@ -23,9 +24,7 @@ source $TOPDIR/configure_easybuild echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used -# PR #16531: Nextflow-22.10.1.eb -${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out} +${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." From 2b6d630c36eb078a268548b812af98143c15da8f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 19:14:15 +0200 Subject: [PATCH 18/30] update test workflow to test with version 2023.06 of EESSI pilot repo --- .github/workflows/test_eessi.yml | 17 ++++++++--------- .github/workflows/tests_scripts.yml | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 04195dd619..65415f87fc 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: EESSI_VERSION: - - 2021.12 + - 2023.06 EESSI_SOFTWARE_SUBDIR: - aarch64/generic - aarch64/graviton2 @@ -20,6 +20,8 @@ jobs: - x86_64/intel/haswell - x86_64/intel/skylake_avx512 - x86_64/generic + EASYSTACK_FILE: + - eessi-2023.06-eb-4.7.2-2021a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 @@ -41,7 +43,7 @@ jobs: export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" - ./check_missing_installations.sh + ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | @@ -52,18 +54,15 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)" - echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml - tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml + echo "modify easystack file by adding a missing package (GCC/8.3.0)" + echo " - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}} + tail -n 5 ${{matrix.EASYSTACK_FILE}} # note, check_missing_installations.sh exits 1 if a package was # missing, which is intepreted as false (exit code based, not # boolean logic), hence when the script exits 0 if no package was # missing it is interpreted as true, thus the test did not capture # the missing package - if ./check_missing_installations.sh; then + if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then echo "did NOT capture missing package; test FAILED" exit 1 else diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 5c0b3893ae..8ad4928993 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -92,6 +92,6 @@ jobs: # since create_directory_tarballs.sh must be accessible from within build container cp -a * /tmp/ cd /tmp - ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12 + ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz From 074d7a1847ab2353e43045978dec6acd72a95cb1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 20:06:08 +0200 Subject: [PATCH 19/30] update check-result.sh to state tested in NESSI PR125 --- bot/check-result.sh | 429 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 351 insertions(+), 78 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index e28ae4e38e..04a23962e2 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -4,8 +4,8 @@ # Intended use is that it is called by a (batch) job running on a compute # node. # -# This script is part of the EESSI compatibility layer, see -# https://github.com/EESSI/compatibility-layer.git +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git # # author: Thomas Roeblitz (@trz42) # @@ -38,6 +38,20 @@ TOPDIR=$(dirname $(realpath $0)) source ${TOPDIR}/../scripts/utils.sh source ${TOPDIR}/../scripts/cfg_files.sh +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + echo_red "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +else + echo "bot/check-result.sh: showing ${JOB_CFG_FILE} from software-layer side" + cat ${JOB_CFG_FILE} + + echo "bot/check-result.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" + cfg_load ${JOB_CFG_FILE} +fi + display_help() { echo "usage: $0 [OPTIONS]" echo " OPTIONS:" @@ -88,46 +102,61 @@ job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" -GP_error='ERROR: ' -grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && ERROR=1 || ERROR=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +ERROR=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_error='ERROR: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") + [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_failed='FAILED: ' -grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && FAILED=1 || FAILED=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +FAILED=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_failed='FAILED: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") + [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_req_missing=' required modules missing:' -grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && MISSING=1 || MISSING=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +MISSING=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_req_missing=' required modules missing:' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}") + [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_no_missing='No missing modules!' -grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +NO_MISSING=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_no_missing='No missing installations' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}") + [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_tgz_created="tar.gz created!" +TGZ=-1 TARBALL= -grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) -if [[ $? -eq 0 ]]; then - TGZ=1 - TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') -else - TGZ=0 +if [[ ${SLURM} -eq 1 ]]; then + GP_tgz_created="\.tar\.gz created!" + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) + if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') + else + TGZ=0 + fi + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" [[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" @@ -147,64 +176,308 @@ if [[ ${SLURM} -eq 1 ]] && \ [[ ${TGZ} -eq 1 ]] && \ [[ ! -z ${TARBALL} ]]; then # SUCCESS - echo "[RESULT]" > ${job_result_file} - echo "summary = :grin: SUCCESS" >> ${job_result_file} - echo "details =" >> ${job_result_file} + status="SUCCESS" + summary=":grin: SUCCESS" else # FAILURE - echo "[RESULT]" > ${job_result_file} - echo "summary = :cry: FAILURE" >> ${job_result_file} - echo "details =" >> ${job_result_file} + status="FAILURE" + summary=":cry: FAILURE" fi -function succeeded() { - echo " :heavy_check_mark: ${1}" +### Example details/descriptions +# Note, final string must not contain any line breaks. Below examples include +# line breaks for the sake of readability. +#
+# :cry: FAILURE _(click triangle for detailed information)_ +# Details:
+#     :heavy_check_mark: job output file slurm-470503.out
+#     :heavy_multiplication_x: found message matching ERROR:
+#     :heavy_multiplication_x: found message matching FAILED:
+#     :heavy_multiplication_x: found message matching required modules missing:
+#     :heavy_check_mark: found message(s) matching No missing installations
+#     :heavy_check_mark: found message matching tar.gz created!
+# Artefacts: +#
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • +#
    +# +#
    +# :grin: SUCCESS _(click triangle for detailed information)_ +# Details:
    +#     :heavy_check_mark: job output file slurm-470503.out
    +#     :heavy_check_mark: found message matching ERROR:
    +#     :heavy_check_mark: found message matching FAILED:
    +#     :heavy_check_mark: found message matching required modules missing:
    +#     :heavy_check_mark: found message(s) matching No missing installations
    +#     :heavy_check_mark: found message matching tar.gz created!
    +# Artefacts: +#
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • +#
    +# +#
    +# :grin: SUCCESS _(click triangle for detailed information)_ +#
    +#
    _Details_
    +#
    +# :heavy_check_mark: job output file slurm-4682.out
    +# :heavy_check_mark: no message matching ERROR:
    +# :heavy_check_mark: no message matching FAILED:
    +# :heavy_check_mark: no message matching required modules missing:
    +# :heavy_check_mark: found message(s) matching No missing installations
    +# :heavy_check_mark: found message matching tar.gz created!
    +#
    +#
    _Artefacts_
    +#
    +#
    +# eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz +# size: 234 MiB (245366784 bytes)
    +# entries: 1234
    +# modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    +# 
    +#           GCC/9.3.0.lua
    
+#           GCC/10.3.0.lua
    
+#           OpenSSL/1.1.lua
+#
    +# software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ +# 
    +#           GCC/9.3.0/
    
+#           CMake/3.20.1-GCCcore-10.3.0/
    
+#           OpenMPI/4.1.1-GCC-10.3.0/
+#
    +# other under _2023.04/software/linux/x86_64/intel/cascadelake/_ +# 
    +#           .lmod/cache/spiderT.lua
    
+#           .lmod/cache/spiderT.luac_5.1
    
+#           .lmod/cache/timestamp
+#
    +#
    +#
    +#
    +#
    +# +### + +# construct and write complete PR comment details: implements third alternative +comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT____ARTEFACTS_FMT__
    " +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" +comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " +comment_success_item_fmt=":heavy_check_mark: __ITEM__" +comment_failure_item_fmt=":heavy_multiplication_x: __ITEM__" +comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " +comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " + +function print_br_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}
    " } -function failed() { - echo " :heavy_multiplication_x: ${1}" +function print_br_item2() { + format="${1}" + item="${2}" + item2="${3}" + format1="${format//__ITEM__/${item}}" + echo -n "${format1//__ITEM2__/${item2}}
    " } -if [[ ${SLURM} -eq 1 ]]; then - succeeded "job output file ${job_out}" >> ${job_result_file} -else - failed "no job output file matching ${GP_slurm_out}" >> ${job_result_file} -fi +function print_code_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}" +} -if [[ ${ERROR} -eq 0 ]]; then - succeeded "no message matching ${GP_error}" >> ${job_result_file} -else - failed "found message matching ${GP_error}" >> ${job_result_file} -fi +function print_dd_item() { + format="${1}" + item="${2}" + echo -n "
    ${format//__ITEM__/${item}}
    " +} -if [[ ${FAILED} -eq 0 ]]; then - succeeded "no message matching ${GP_failed}" >> ${job_result_file} -else - failed "found message matching ${GP_failed}" >> ${job_result_file} -fi +function print_list_item() { + format="${1}" + item="${2}" + echo -n "
  • ${format//__ITEM__/${item}}
    • " +} -if [[ ${MISSING} -eq 0 ]]; then - succeeded "no message matching ${GP_req_missing}" >> ${job_result_file} -else - failed "found message matching ${GP_req_missing}" >> ${job_result_file} -fi +function print_pre_item() { + format="${1}" + item="${2}" + echo -n "
    ${format//__ITEM__/${item}}
    " +} -if [[ ${NO_MISSING} -eq 1 ]]; then - succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} -else - failed "no message matching ${GP_no_missing}" >> ${job_result_file} -fi +function success() { + format="${comment_success_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} -if [[ ${TGZ} -eq 1 ]]; then - succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} -else - failed "no message matching ${GP_tgz_created}" >> ${job_result_file} -fi +function failure() { + format="${comment_failure_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} + +function add_detail() { + actual=${1} + expected=${2} + success_msg="${3}" + failure_msg="${4}" + if [[ ${actual} -eq ${expected} ]]; then + success "${success_msg}" + else + failure "${failure_msg}" + fi +} + +echo "[RESULT]" > ${job_result_file} +echo -n "comment_description = " >> ${job_result_file} + +# construct values for placeholders in comment_template: +# - __SUMMARY_FMT__ -> variable $comment_summary +# - __DETAILS_FMT__ -> variable $comment_details +# - __ARTEFACTS_FMT__ -> variable $comment_artefacts -echo "artefacts =" >> ${job_result_file} +comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" +# first construct comment_details_list, abbreviated CoDeList +# then use it to set comment_details +CoDeList="" + +success_msg="job output file ${job_out}" +failure_msg="no job output file matching ${GP_slurm_out}" +CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_error}" +failure_msg="found message matching ${GP_error}" +CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_failed}" +failure_msg="found message matching ${GP_failed}" +CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_req_missing}" +failure_msg="found message matching ${GP_req_missing}" +CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") + +success_msg="found message(s) matching ${GP_no_missing}" +failure_msg="no message matching ${GP_no_missing}" +CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") + +success_msg="found message matching ${GP_tgz_created}" +failure_msg="no message matching ${GP_tgz_created}" +CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") + +comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" + + +# first construct comment_artefacts_list, abbreviated CoArList +# then use it to set comment_artefacts +CoArList="" + +# TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then - echo " ${TARBALL}" >> ${job_result_file} + # TODO add tarball details: size, num entries, modules, software pkgs, misc + #
    + #
    + # eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz + # size: 234 MiB (245366784 bytes)
    + # entries: 1234
    + # modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    + # 
    +    #       GCC/9.3.0.lua
    
+    #       GCC/10.3.0.lua
    
+    #       OpenSSL/1.1.lua
+    #
    + # software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ + # 
    +    #       GCC/9.3.0/
    
+    #       CMake/3.20.1-GCCcore-10.3.0/
    
+    #       OpenMPI/4.1.1-GCC-10.3.0/
+    #
    + # other under _2023.04/software/linux/x86_64/intel/cascadelake/_ + # 
    +    #       .lmod/cache/spiderT.lua
    
+    #       .lmod/cache/spiderT.luac_5.1
    
+    #       .lmod/cache/timestamp
+    #
    + #
    + #
    + size="$(stat --dereference --printf=%s ${TARBALL})" + size_mib=$((${size} >> 20)) + tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX) + tar tf ${TARBALL} > ${tmpfile} + entries=$(cat ${tmpfile} | wc -l) + # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE + # 2023.04/software/linux/x86_64/intel/skylake_avx512 + # repo_version = 2022.11 + # software + # os_type = linux + # software_subdir = aarch64/generic + repo_version=$(cfg_get_value "repository" "repo_version") + os_type=$(cfg_get_value "architecture" "os_type") + software_subdir=$(cfg_get_value "architecture" "software_subdir") + prefix="${repo_version}/software/${os_type}/${software_subdir}" + modules_entries=$(grep "${prefix}/modules" ${tmpfile}) + software_entries=$(grep "${prefix}/software" ${tmpfile}) + lmod_entries=$(grep "${prefix}/.lmod/cache" ${tmpfile}) + other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") + other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) + modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) + software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) + lmod_shortened=$(echo "${lmod_entries}" | sed -e "s@${prefix}/@@") + + artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" + CoArList="" + CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" + CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})" + CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" + CoArList="${CoArList}
    "
+    if [[ ! -z ${modules} ]]; then
+        while IFS= read -r mod ; do
+            CoArList="${CoArList}$(print_br_item '__ITEM__' ${mod})"
+        done <<< "${modules}"
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
+    fi
+    CoArList="${CoArList}
    " + CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" + CoArList="${CoArList}
    "
+    if [[ ! -z ${software_pkgs} ]]; then
+        while IFS= read -r sw_pkg ; do
+            CoArList="${CoArList}$(print_br_item '__ITEM__' ${sw_pkg})"
+        done <<< "${software_pkgs}"
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+    fi
+    CoArList="${CoArList}
    " + CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})" + CoArList="${CoArList}
    "
+    if [[ ! -z ${other_shortened} ]]; then
+        while IFS= read -r other ; do
+            CoArList="${CoArList}$(print_br_item '__ITEM__' ${other})"
+        done <<< "${other_shortened}"
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
+    fi
+    CoArList="${CoArList}
    " +else + CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')" fi +comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" +comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}" +comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" + +# now put all pieces together creating comment_details from comment_template +comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} +comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} +comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} + +echo "${comment_description}" >> ${job_result_file} + +# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts +# - this should make use of subsequent steps such as deploying a tarball more +# efficient +echo "status = ${status}" >> ${job_result_file} +echo "artefacts = " >> ${job_result_file} +echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} + exit 0 From 3c43a92524f4f8f6d5db167c8a58cedc273d51a0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 20:09:00 +0200 Subject: [PATCH 20/30] check for missing installations requires a few updates for check-result.sh --- check_missing_installations.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 926f475903..3627d1d0b5 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -1,18 +1,20 @@ #!/bin/bash # -# Script to check for missing installations in EESSI pilot software stack (version 2021.12) +# Script to check for missing installations in EESSI pilot software stack (version 2023.06) # # author: Kenneth Hoste (@boegel) +# author: Thomas Roeblitz (@trz42) # # license: GPLv2 # TOPDIR=$(dirname $(realpath $0)) -if [ -z ${EESSI_PILOT_VERSION} ]; then - echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2 +if [ $# -ne 1 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi +easystack=$1 LOCAL_TMPDIR=$(mktemp -d) @@ -21,12 +23,13 @@ source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." -ok_msg="No missing installations, party time!" -fail_msg="On no, some installations are still missing, how did that happen?!" eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used -# PR #16531: Nextflow-22.10.1.eb -${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} +exit_code=${PIPESTATUS[0]} + +ok_msg="Command 'eb --missing ...' succeeded, analysing output..." +fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" +check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" # the above assesses the installed software for each easyconfig provided in # the easystack file and then print messages such as @@ -37,8 +40,11 @@ ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experim # output does not contain any line with ` required modules missing:` grep " required modules missing:" ${eb_missing_out} > /dev/null +exit_code=$? # if grep returns 1 (` required modules missing:` was NOT found), we set # MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 -[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +[[ ${exit_code} -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +ok_msg="No missing installations, party time!" +fail_msg="On no, some installations are still missing, how did that happen?!" check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}" From 5a0294e0bf1c553a108ab5ceba5dff07b90be8d6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 20:09:43 +0200 Subject: [PATCH 21/30] make sure experimental EasyBuild features can be used --- configure_easybuild | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure_easybuild b/configure_easybuild index 19b2d7454b..8e79f3a35a 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -35,3 +35,6 @@ fi export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER export EASYBUILD_MODULE_EXTENSIONS=1 + +# need to enable use of experimental features, since we're using easystack files +export EASYBUILD_EXPERIMENTAL=1 From 4110548e9280795f32a4c60ed979430217a50771 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 10:35:44 +0200 Subject: [PATCH 22/30] removed OpenSSL from filter deps --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index ad34487902..23f3920154 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -26,7 +26,7 @@ fi # note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49) # filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104) # problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency) -DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,OpenSSL,util-linux,XZ,zlib +DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib # For aarch64 we need to also filter out Yasm. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190 if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then From f709a8a72d425acfa02dd9d3e8114af99b796b1f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 10:43:56 +0200 Subject: [PATCH 23/30] let script's return value reflect SUCCESS (0) or FAILURE (1) --- bot/check-result.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/check-result.sh b/bot/check-result.sh index 04a23962e2..ec58fea557 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -480,4 +480,6 @@ echo "status = ${status}" >> ${job_result_file} echo "artefacts = " >> ${job_result_file} echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} +test "${status}" == "SUCCESS" +exit $? exit 0 From 5cfb4e1510cd8a0c93824af33ff5c4a805e3bd11 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 10:52:06 +0200 Subject: [PATCH 24/30] use test to check if slurm-*.out exists + fix reporting message --- bot/check-result.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index ec58fea557..6c23cae680 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -96,11 +96,14 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" GP_slurm_out="slurm-${SLURM_JOB_ID}.out" -job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") -[[ $? -eq 0 ]] && SLURM=1 || SLURM=0 -# have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" -[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" +if [[ -f ${GP_slurm_out} ]]; then + SLURM=1 + [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${GP_slurm_out}"'" +else + SLURM=0 + [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${GP_slurm_out}"' NOT found" +fi ERROR=-1 if [[ ${SLURM} -eq 1 ]]; then From 7478e7d65ab2836135e62fc6f866d84d27a7b0a7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 11:09:34 +0200 Subject: [PATCH 25/30] more polishing to address multiple comments/suggestions --- bot/check-result.sh | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 6c23cae680..33542d61ed 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,7 +152,7 @@ if [[ ${SLURM} -eq 1 ]]; then grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) if [[ $? -eq 0 ]]; then TGZ=1 - TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') + TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@') else TGZ=0 fi @@ -162,7 +162,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" -[[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" +[[ ${VERBOSE} -ne 0 ]] && echo " : ()" [[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" @@ -377,25 +377,28 @@ CoArList="" # TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then - # TODO add tarball details: size, num entries, modules, software pkgs, misc + # Example of the detailed information for a tarball. The actual result MUST be a + # single line (no '\n') or it would break the structure of the markdown table + # that holds status updates of a bot job. + # #
    #
    - # eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz + # eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz # size: 234 MiB (245366784 bytes)
    # entries: 1234
    - # modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    + # modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_
    # 
         #       GCC/9.3.0.lua
    
     #       GCC/10.3.0.lua
    
     #       OpenSSL/1.1.lua
     #
    - # software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ + # software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_ # 
         #       GCC/9.3.0/
    
     #       CMake/3.20.1-GCCcore-10.3.0/
    
     #       OpenMPI/4.1.1-GCC-10.3.0/
     #
    - # other under _2023.04/software/linux/x86_64/intel/cascadelake/_ + # other under _2023.06/software/linux/x86_64/intel/cascadelake/_ # 
         #       .lmod/cache/spiderT.lua
    
     #       .lmod/cache/spiderT.luac_5.1
    
@@ -409,23 +412,25 @@ if [[ ! -z ${TARBALL} ]]; then
     tar tf ${TARBALL} > ${tmpfile}
     entries=$(cat ${tmpfile} | wc -l)
     # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
-    # 2023.04/software/linux/x86_64/intel/skylake_avx512
-    # repo_version = 2022.11
-    # software
+    # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512
+    # cfg/job.cfg contains (only the attributes to be used are shown below):
+    # [repository]
+    # repo_version = 2023.06
+    # [architecture]
     # os_type = linux
-    # software_subdir = aarch64/generic
+    # software_subdir = x86_64/intel/skylake_avx512
     repo_version=$(cfg_get_value "repository" "repo_version")
     os_type=$(cfg_get_value "architecture" "os_type")
     software_subdir=$(cfg_get_value "architecture" "software_subdir")
     prefix="${repo_version}/software/${os_type}/${software_subdir}"
+
+    # extract directories/entries from tarball content
     modules_entries=$(grep "${prefix}/modules" ${tmpfile})
     software_entries=$(grep "${prefix}/software" ${tmpfile})
-    lmod_entries=$(grep "${prefix}/.lmod/cache" ${tmpfile})
     other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
     other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
     modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
     software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
-    lmod_shortened=$(echo "${lmod_entries}" | sed -e "s@${prefix}/@@")
 
     artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})"
     CoArList=""

From 827e686edfc1540d82a4bbf5d069ed9460822efc Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz 
Date: Tue, 20 Jun 2023 11:14:01 +0200
Subject: [PATCH 26/30] remove tmpfile + clarify/improve exit

---
 bot/check-result.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bot/check-result.sh b/bot/check-result.sh
index 33542d61ed..2ffbcd6be6 100755
--- a/bot/check-result.sh
+++ b/bot/check-result.sh
@@ -488,6 +488,11 @@ echo "status = ${status}" >> ${job_result_file}
 echo "artefacts = " >> ${job_result_file}
 echo "${TARBALL}" | sed -e 's/^/    /g' >> ${job_result_file}
 
+# remove tmpfile
+if [[ -f ${tmpfile} ]]; then
+    rm ${tmpfile}
+fi
+
+# exit script with value that reflects overall job result: SUCCESS (0), FAILURE (1)
 test "${status}" == "SUCCESS"
 exit $?
-exit 0

From 9296ed75e52b680bf278aba9ff2fb82ec5318765 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz 
Date: Tue, 20 Jun 2023 11:47:19 +0200
Subject: [PATCH 27/30] polishing examples + comments

---
 bot/check-result.sh | 62 ++++++++++++++++++++-------------------------
 1 file changed, 28 insertions(+), 34 deletions(-)

diff --git a/bot/check-result.sh b/bot/check-result.sh
index 2ffbcd6be6..51e1777bab 100755
--- a/bot/check-result.sh
+++ b/bot/check-result.sh
@@ -20,7 +20,7 @@
 #    - no message ERROR
 #    - no message FAILED
 #    - no message ' required modules missing:'
-#    - one or more of 'No missing modules!'
+#    - one or more of 'No missing installations'
 #    - message regarding created tarball
 #  - FAILED (one of ... implemented as NOT SUCCESS)
 #    - no slurm-JOBID.out file
@@ -188,36 +188,12 @@ else
 fi
 
 ### Example details/descriptions
-# Note, final string must not contain any line breaks. Below examples include
-# line breaks for the sake of readability.
+# Note, final string must not contain any line breaks. Below example include
+# line breaks for the sake of readability. In case of FAILURE, the structure is
+# very similar (incl. information about Artefacts if any was produced), however,
+# under Details some lines will be marked with :heavy_multiplication_x:
 # 
    
-#   :cry: FAILURE _(click triangle for detailed information)_
-#   Details:
    
-#       :heavy_check_mark: job output file slurm-470503.out
    
-#       :heavy_multiplication_x: found message matching ERROR: 
    
-#       :heavy_multiplication_x: found message matching FAILED: 
    
-#       :heavy_multiplication_x: found message matching  required modules missing:
    
-#       :heavy_check_mark: found message(s) matching No missing installations
    
-#       :heavy_check_mark: found message matching tar.gz created!
    
-#   Artefacts:
-#   
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • 
-# 
    
-#
-# 
    
-#   :grin: SUCCESS _(click triangle for detailed information)_
-#   Details:
    
-#       :heavy_check_mark: job output file slurm-470503.out
    
-#       :heavy_check_mark: found message matching ERROR: 
    
-#       :heavy_check_mark: found message matching FAILED: 
    
-#       :heavy_check_mark: found message matching  required modules missing:
    
-#       :heavy_check_mark: found message(s) matching No missing installations
    
-#       :heavy_check_mark: found message matching tar.gz created!
    
-#   Artefacts:
-#   
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • 
-# 
    
-#
-# 
    
-#   :grin: SUCCESS _(click triangle for detailed information)_
+#   :grin: SUCCESS _(click triangle for details)_
 #   
    
 #     
    _Details_
    
 #     
    
@@ -231,22 +207,22 @@ fi
 #     
    _Artefacts_
    
 #     
    
 #       
    
-#         eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz
+#         eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz
 #         size: 234 MiB (245366784 bytes)
    
 #         entries: 1234
    
-#         modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    
+#         modules under _2023.06/software/linux/x86_64/generic/modules/all/_
    
 #         
     #           GCC/9.3.0.lua
    
 #           GCC/10.3.0.lua
    
 #           OpenSSL/1.1.lua
 #         
    
-#         software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_
+#         software under _2023.06/software/linux/x86_64/generic/software/_
 #         
     #           GCC/9.3.0/
    
 #           CMake/3.20.1-GCCcore-10.3.0/
    
 #           OpenMPI/4.1.1-GCC-10.3.0/
 #         
    
-#         other under _2023.04/software/linux/x86_64/intel/cascadelake/_
+#         other under _2023.06/software/linux/x86_64/generic/_
 #         
     #           .lmod/cache/spiderT.lua
    
 #           .lmod/cache/spiderT.luac_5.1
    
@@ -257,6 +233,24 @@ fi
 #   
    
 # 
    
 #
+# 
    
+#   :cry: FAILURE _(click triangle for details)_
+#   
    
+#     
    _Details_
    
+#     
    
+#       :heavy_check_mark: job output file slurm-4682.out
    
+#       :heavy_multiplication_x: no message matching ERROR: 
    
+#       :heavy_check_mark: no message matching FAILED: 
    
+#       :heavy_multiplication_x: no message matching  required modules missing:
    
+#       :heavy_check_mark: found message(s) matching No missing installations
    
+#       :heavy_check_mark: found message matching tar.gz created!
    
+#     
    
+#     
    _Artefacts_
    
+#     
    
+#       No artefacts were created or found.
+#     
    
+#   
    
+# 
    
 ###
 
 # construct and write complete PR comment details: implements third alternative

From 78a9dc8e9e67dc998c39e6f74f06b72117e0f22d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste 
Date: Tue, 20 Jun 2023 13:50:37 +0200
Subject: [PATCH 28/30] make sure that $job_out is defined in
 bot/check-result.sh

---
 bot/check-result.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/bot/check-result.sh b/bot/check-result.sh
index 51e1777bab..182b8555a8 100755
--- a/bot/check-result.sh
+++ b/bot/check-result.sh
@@ -95,14 +95,14 @@ job_dir=${PWD}
 
 [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
 
-GP_slurm_out="slurm-${SLURM_JOB_ID}.out"
-[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'"
-if  [[ -f ${GP_slurm_out} ]]; then
+job_out="slurm-${SLURM_JOB_ID}.out"
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
+if  [[ -f ${job_out} ]]; then
     SLURM=1
-    [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${GP_slurm_out}"'"
+    [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 else
     SLURM=0
-    [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${GP_slurm_out}"' NOT found"
+    [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
 fi
 
 ERROR=-1
@@ -339,7 +339,7 @@ comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 CoDeList=""
 
 success_msg="job output file ${job_out}"
-failure_msg="no job output file matching ${GP_slurm_out}"
+failure_msg="no job output file ${job_out}"
 CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching ${GP_error}"

From 4cc8f4135fd030f942950af70559a40aa43143cd Mon Sep 17 00:00:00 2001
From: Kenneth Hoste 
Date: Tue, 20 Jun 2023 20:53:32 +0200
Subject: [PATCH 29/30] don't run 'eb --missing' in install script, since it
 confuses the bot's check-result script because of 'required modules missing'
 messages in job output

---
 EESSI-pilot-install-software.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index f2d50dfe91..287688a61f 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -151,7 +151,7 @@ for eb_version in '4.7.2'; do
         if [ -f ${es} ]; then
             echo_green "Feeding easystack file ${es} to EasyBuild..."
 
-            ${EB} --easystack ${TOPDIR}/${es} --missing && ${EB} --easystack ${TOPDIR}/${es} --robot
+            ${EB} --easystack ${TOPDIR}/${es} --robot
 
             $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es}
         else

From 5d7e367e32af6e8d6765b2c3a88451901043237f Mon Sep 17 00:00:00 2001
From: Kenneth Hoste 
Date: Thu, 22 Jun 2023 17:41:02 +0200
Subject: [PATCH 30/30] use correct aarch64 CPU targets for tests that rely on
 EESSI pilot 2023.06

---
 .github/workflows/test_eessi.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 65415f87fc..3d18e56816 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -13,8 +13,8 @@ jobs:
         - 2023.06
         EESSI_SOFTWARE_SUBDIR:
         - aarch64/generic
-        - aarch64/graviton2
-        - aarch64/graviton3
+        - aarch64/neoverse_n1
+        - aarch64/neoverse_v1
         - x86_64/amd/zen2
         - x86_64/amd/zen3
         - x86_64/intel/haswell