Skip to content

debug failing CI after lmod cache is not updated by bot build jobs #514

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

debug failing CI after lmod cache is not updated by bot build jobs #514

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a0dc281
fix comment for creating lmod rc code block
bedroge Mar 26, 2024
b2570ed
don't update lmod cache here anymore
bedroge Mar 26, 2024
cdf78d1
adding a bit more progress output to install_cuda_host_injections.sh
truib Mar 26, 2024
2bd1ada
added more debug messages
truib Mar 26, 2024
db91275
use older apptainer (1.2.4)
truib Mar 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions EESSI-install-software.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ fi

### add packages here

echo ">> Creating/updating Lmod cache..."
echo ">> Creating/updating Lmod RC file..."
export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod"
lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua"
lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?)
Expand All @@ -265,7 +265,5 @@ if [ ! -f "$lmod_sitepackage_file" ] || [ "${sitepackage_changed}" == '0' ]; the
check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file"
fi

$TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}

echo ">> Cleaning up ${TMPDIR}..."
rm -r ${TMPDIR}
7 changes: 5 additions & 2 deletions install_apptainer_ubuntu.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ set -e
# see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
sudo apt-get install alien
alien --version
apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm}
#apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
#curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm}
#/pub/archive/epel/8.8/Everything/x86_64/Packages/a
apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
curl -OL https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/${apptainer_rpm}
sudo alien -d ${apptainer_rpm}
sudo apt install ./apptainer*.deb
apptainer --version
Expand Down
13 changes: 13 additions & 0 deletions scripts/gpu_support/nvidia/install_cuda_host_injections.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,13 @@ else
fi

if ! command -v "eb" &>/dev/null; then
env | sort
module --version
declare -f module
module list
ls ${EESSI_SOFTWARE_PATH}
ls ${cuda_install_parent}

echo_yellow "Attempting to load an EasyBuild module to do actual install"
module load EasyBuild
# There are some scenarios where this may fail
Expand All @@ -165,12 +172,14 @@ else
error="${error}Please re-run this script with the 'eb' command available."
fatal_error "${error}"
fi
echo_yellow "EasyBuild module loaded"
fi

cuda_easyconfig="CUDA-${install_cuda_version}.eb"

# Check the easyconfig file is available in the release
# (eb search always returns 0, so we need a grep to ensure a usable exit code)
echo_yellow "Searching for easyconfig '${cuda_easyconfig}'"
eb --search ^${cuda_easyconfig}|grep CUDA > /dev/null 2>&1
# Check the exit code
if [ $? -ne 0 ]; then
Expand All @@ -185,6 +194,7 @@ else
error="${error}${available_cuda_easyconfigs}"
fatal_error "${error}"
fi
echo_yellow "Search for easyconfig '${cuda_easyconfig}' done"

# We need the --rebuild option, as the CUDA module may or may not be on the
# `MODULEPATH` yet. Even if it is, we still want to redo this installation
Expand All @@ -195,10 +205,13 @@ else
extra_args="--rebuild --installpath-modules=${tmpdir}"

# We don't want hooks used in this install, we need a vanilla CUDA installation
echo_yellow "Creating empty hooks file"
touch "$tmpdir"/none.py
# shellcheck disable=SC2086 # Intended splitting of extra_args
echo_yellow "Running 'eb --prefix=$tmpdir ${extra_args} --accept-eula-for=CUDA --hooks=$tmpdir/none.py --installpath=${cuda_install_parent}/ ${cuda_easyconfig}'"
eb --prefix="$tmpdir" ${extra_args} --accept-eula-for=CUDA --hooks="$tmpdir"/none.py --installpath="${cuda_install_parent}"/ "${cuda_easyconfig}"
ret=$?
echo_yellow "eb command finished with exit code '$ret'"
if [ $ret -ne 0 ]; then
eb_last_log=$(unset EB_VERBOSE; eb --last-log)
cp -a ${eb_last_log} .
Expand Down