@@ -17,4 +17,58 @@ function spushd(){
1717# safe popd
1818function spopd(){
1919 popd > /dev/null || die " popd failed :("
20- }
20+ }
21+
22+ # Retry a command multiple times until it succeeds, with escalating
23+ # delay between attempts.
24+ # Delay is 2 * n + random up to 30s, then 30s + random after that.
25+ # For large numbers of retries the max delay is effectively the retry
26+ # in minutes.
27+ # Based on:
28+ # https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
29+ # but now quite heavily modified.
30+ function retry {
31+ # Maxiumum amount of fixed delay between attempts
32+ # a random value will still be added.
33+ local -r MAX_BACKOFF=30
34+
35+ if [[ ${# } -lt 2 ]]; then
36+ echo " retry usage: retry <retries> <command>"
37+ exit 1
38+ fi
39+
40+ local retries=$1
41+ shift
42+
43+ if ! [[ ${retries} =~ ^[0-9\. ]* $ ]]; then
44+ echo " Invalid number of retries: ${retries} for command '${* } '"
45+ exit 1
46+ fi
47+
48+ local count=0
49+ until " $@ " ; do
50+ # Command failed, otherwise until would have skipped the loop
51+
52+ # Store return code so it can be reported to the user
53+ exit=$?
54+ count=$(( count + 1 ))
55+ if [ " ${count} " -lt " ${retries} " ; then
56+ # There are still retries left, calculate delay and notify user.
57+ backoff=$(( 2 * count))
58+ if [[ " ${backoff} " -gt " ${MAX_BACKOFF} " ; then
59+ backoff=${MAX_BACKOFF}
60+ fi ;
61+
62+ # Add a random amount to the delay to prevent competing processes
63+ # from re-colliding.
64+ wait=$(( backoff + (RANDOM % count) ))
65+ echo " '${* } ' Retry $count /$retries exited $exit , retrying in $wait seconds..."
66+ sleep $wait
67+ else
68+ # Out of retries :(
69+ echo " Retry $count /$retries exited $exit , no more retries left."
70+ return $exit
71+ fi
72+ done
73+ return 0
74+ }
0 commit comments