Skip to content

Commit cf64753

Browse files
casparvlcasparvl
authored
Merge pull request #60 from bedroge/zstd_tarballs
add possibility for generating zstd-compressed tarballs
2 parents 6303643 + bfa8884 commit cf64753

File tree

3 files changed

+38
-22
lines changed

3 files changed

+38
-22
lines changed

‎bot/build.sh‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -265,16 +265,24 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
265265
TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
266266

267267
timestamp=$(date +%s)
268+
# determine compression/extension for tarball, check in order of preference
269+
if [[ -x "$(command -v zstd)" ]]; then
270+
tarball_extension="tar.zst"
271+
elif [[ -x "$(command -v gzip)" ]]; then
272+
tarball_extension="tar.gz"
273+
else
274+
tarball_extension="tar"
275+
fi
268276
# to set EESSI_VERSION we need to source init/eessi_defaults now
269277
source $software_layer_dir/init/eessi_defaults
270278
# Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we
271279
# append the project (subdirectory) name to the end tarball name. This is information
272280
# then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is
273281
# appended
274282
if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then
275-
export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
283+
export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
276284
else
277-
export TGZ=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
285+
export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
278286
fi
279287

280288
# Export EESSI_DEV_PROJECT to use it (if needed) when making tarball
@@ -288,8 +296,8 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}
288296
TMP_IN_CONTAINER=/tmp
289297
echo "Executing command to create tarball:"
290298
echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
291-
echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
299+
echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}"
292300
$software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
293-
-- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
301+
-- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}
294302

295303
exit 0

‎bot/check-build.sh‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
# - SUCCESS (all of)
1919
# - working directory contains slurm-JOBID.out file
20-
# - working directory contains eessi*tar.gz
20+
# - working directory contains eessi*tar*
2121
# - no message FATAL
2222
# - no message ERROR
2323
# - no message FAILED
@@ -165,19 +165,19 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
165165
fi
166166

167167
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
168-
TGZ=-1
168+
TARBALL_CREATED=-1
169169
TARBALL=
170170
if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
171-
GP_tgz_created="\.tar\.gz created!"
172-
grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
171+
GP_tarball_created="\.tar.* created!"
172+
grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tarball_created}" | sort -u)
173173
if [[ $? -eq 0 ]]; then
174-
TGZ=1
174+
TARBALL_CREATED=1
175175
TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@')
176176
else
177-
TGZ=0
177+
TARBALL_CREATED=0
178178
fi
179179
# have to be careful to not add searched for pattern into slurm out file
180-
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
180+
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tarball_created}"'"
181181
[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
182182
fi
183183
fi
@@ -190,7 +190,7 @@ fi
190190
[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
191191
[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
192192
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
193-
[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
193+
[[ ${VERBOSE} -ne 0 ]] && echo " TARBALL_CREATED: $([[ $TARBALL -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
194194
fi
195195

196196
# Here, we try to do some additional analysis on the output file
@@ -219,7 +219,7 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
219219
[[ ${FAILED} -eq 0 ]] && \
220220
[[ ${MISSING} -eq 0 ]] && \
221221
[[ ${NO_MISSING} -eq 1 ]] && \
222-
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TGZ} -eq 1 ]] && \
222+
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TARBALL_CREATED} -eq 1 ]] && \
223223
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || -n ${TARBALL} ]]; then
224224
# SUCCESS
225225
status="SUCCESS"
@@ -429,9 +429,9 @@ failure_msg="no message matching <code>${GP_no_missing}</code>"
429429
comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
430430

431431
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
432-
success_msg="found message matching <code>${GP_tgz_created}</code>"
433-
failure_msg="no message matching <code>${GP_tgz_created}</code>"
434-
comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
432+
success_msg="found message matching <code>${GP_tarball_created}</code>"
433+
failure_msg="no message matching <code>${GP_tarball_created}</code>"
434+
comment_details_list=${comment_details_list}$(add_detail ${TARBALL_CREATED} 1 "${success_msg}" "${failure_msg}")
435435
fi
436436

437437
# Now, do the actual replacement of __DETAILS_FMT__
@@ -478,7 +478,16 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
478478
size="$(stat --dereference --printf=%s ${TARBALL})"
479479
size_mib=$((${size} >> 20))
480480
tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
481-
tar tf ${TARBALL} > ${tmpfile}
481+
if [[ "${TARBALL}" == *.tar.zst ]]; then
482+
tar --use-compress-program=zstd -tf ${TARBALL} > ${tmpfile}
483+
elif [[ "${TARBALL}" == *.tar.gz ]]; then
484+
tar --use-compress-program=gzip -tf ${TARBALL} > ${tmpfile}
485+
elif [[ "${TARBALL}" == *.tar ]]; then
486+
tar -tf ${TARBALL} > ${tmpfile}
487+
else
488+
echo "ERROR: Unsupported tarball extension!" >&2
489+
exit 1
490+
fi
482491
entries=$(cat ${tmpfile} | wc -l)
483492
# determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
484493
# e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512

‎create_tarball.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ eessi_tmpdir=$1
1212
eessi_version=$2
1313
cpu_arch_subdir=$3
1414
accel_subdir=$4
15-
target_tgz=$5
15+
target_tarball=$5
1616

1717
tmpdir=`mktemp -d`
1818
echo ">> tmpdir: $tmpdir"
@@ -114,10 +114,9 @@ fi
114114

115115
topdir=${cvmfs_repo}/versions/
116116

117-
echo ">> Creating tarball ${target_tgz} from ${topdir}..."
118-
tar cfvz ${target_tgz} -C ${topdir} --files-from=${files_list}
119-
120-
echo ${target_tgz} created!
117+
echo ">> Creating tarball ${target_tarball} from ${topdir}..."
118+
tar cavf ${target_tarball} -C ${topdir} --files-from=${files_list}
119+
echo ${target_tarball} created!
121120

122121
echo ">> Cleaning up tmpdir ${tmpdir}..."
123122
rm -r ${tmpdir}

0 commit comments

Comments
 (0)