From 547386f9180687627ce506cab0cacc91ed808025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 10:08:14 +0100 Subject: [PATCH 1/8] dont fail if grep doesnt find rebuild easystack files --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index b007b80172..2438adbcd9 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -182,7 +182,7 @@ fi # determine if the removal step has to be run # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) -changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/") +changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/" || true) if [[ -z ${changed_easystacks_rebuilds} ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." else From c2fad0032008d0ca73f6603ad7fbae6c2391a6a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 10:08:32 +0100 Subject: [PATCH 2/8] add quotes around variable in if statement --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 2438adbcd9..e9f8ecfbb9 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -183,7 +183,7 @@ fi # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/" || true) -if [[ -z ${changed_easystacks_rebuilds} ]]; then +if [[ -z "${changed_easystacks_rebuilds}" ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." else # prepare directory to store tarball of tmp for removal and build steps From bce1067e0b9962c591804c87ea150e38123822be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 10:13:32 +0100 Subject: [PATCH 3/8] use proper tmp dir for removal step --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index e9f8ecfbb9..0ebd55692e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -192,7 +192,7 @@ else # prepare arguments to eessi_container.sh specific to remove step declare -a REMOVAL_STEP_ARGS=() - REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") + REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_REMOVAL_STEP_DIR}") REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}") # add fakeroot option in order to be able to remove software, see: # https://github.com/EESSI/software-layer/issues/312 From 4a9d9aeedd5fc7dc719bbc9612c6b5e2213a673b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 10:20:16 +0100 Subject: [PATCH 4/8] let the tarball step resume from the right directory --- bot/build.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 0ebd55692e..0d23f59b66 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -247,7 +247,14 @@ declare -a TARBALL_STEP_ARGS=() TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}") # determine temporary directory to resume from -TARBALL_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") +if [[ -z ${REMOVAL_TMPDIR} ]]; then + # no rebuild step was done, so the tarball step should resume from the build directory + BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) + TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") +else + # a removal step was done, so resume from its temporary directory (which was also used for the build step) + TARBALL_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") +fi timestamp=$(date +%s) # to set EESSI_VERSION we need to source init/eessi_defaults now From 2ba8513f770f2f23961a05bc64b98488e4aaaa8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 11:14:05 +0100 Subject: [PATCH 5/8] group grep and true commands Co-authored-by: Kenneth Hoste --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 0d23f59b66..ff933a447d 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -182,7 +182,7 @@ fi # determine if the removal step has to be run # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) -changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/" || true) +changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | (grep "/rebuilds/" || true)) if [[ -z "${changed_easystacks_rebuilds}" ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." else From 0e38a4769d7db0e893f9a4eb058ec9333a0e648e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 11:16:35 +0100 Subject: [PATCH 6/8] add comment about true command --- bot/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index ff933a447d..dcc61c19d4 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -182,6 +182,8 @@ fi # determine if the removal step has to be run # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) +# the true at the end of the next command is important: grep will expectedly return 1 if there is no easystack file being added under rebuilds, +# but due to "set -e" the entire script would otherwise fail changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | (grep "/rebuilds/" || true)) if [[ -z "${changed_easystacks_rebuilds}" ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." From 80609663bf16cf013df9922c2229976c19e821df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 16:08:52 +0100 Subject: [PATCH 7/8] add quotes around changed_easystacks variable --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 470663e45a..e2d7cf8c51 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -204,7 +204,7 @@ ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12 # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') -if [ -z ${changed_easystacks} ]; then +if [ -z "${changed_easystacks}" ]; then echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here else From 70935ea4293d0e7a94b99f573a77fe74b4f68fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 16:13:07 +0100 Subject: [PATCH 8/8] first process rebuilds, then new installations --- EESSI-install-software.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index e2d7cf8c51..a905d966f6 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -208,7 +208,11 @@ if [ -z "${changed_easystacks}" ]; then echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here else - for easystack_file in ${changed_easystacks}; do + # first process rebuilds, if any, then easystack files for new installations + # "|| true" is used to make sure that the grep command always returns success + rebuild_easystacks=$(echo "${changed_easystacks}" | (grep "/rebuilds/" || true)) + new_easystacks=$(echo "${changed_easystacks}" | (grep -v "/rebuilds/" || true)) + for easystack_file in ${rebuild_easystacks} ${new_easystacks}; do echo -e "Processing easystack file ${easystack_file}...\n\n"