Retry function added to helper library

This function allows a command to be retried a limited
number of times with an escalating delay between attempts.

Related: conjurinc/ops#423

Author: hughsaunders

README.md

@@ -136,6 +136,7 @@ files within it's directory.
         <ol>
           <li><b>die</b>: print message and exit 1</li>
           <li><b>spushd/spopd</b>: Safe verisons of pushd & popd that call die if the push/pop fails, they also drop stdout. </li>
+          <li><b>retry</b>: Retry a command until it succeeds up to a user specified maximum number of attempts. Escalating delay between attempts.</li>
         </ol>
       </td>
     </tr>

helpers/lib

@@ -18,3 +18,57 @@ function spushd(){
 function spopd(){
     popd >/dev/null || die "popd failed :("
 }
+
+# Retry a command multiple times until it succeeds, with escalating
+# delay between attempts.
+# Delay is 2 * n + random up to 30s, then 30s + random after that.
+# For large numbers of retries the max delay is effectively the retry
+# in minutes.
+# Based on:
+# https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
+# but now quite heavily modified.
+function retry {
+    # Maxiumum amount of fixed delay between attempts
+    # a random value will still be added.
+    local -r MAX_BACKOFF=30
+
+    if [[ ${#} -lt 2 ]]; then
+        echo "retry usage: retry <retries> <command>"
+        exit 1
+    fi
+
+    local retries=$1
+    shift
+
+    if ! [[ ${retries} =~ ^[0-9\.]*$ ]]; then
+        echo "Invalid number of retries: ${retries} for command '${*}'".
+        exit 1
+    fi
+
+    local count=0
+    until "$@"; do
+        # Command failed, otherwise until would have skipped the loop
+
+        # Store return code so it can be reported to the user
+        exit=$?
+        count=$((count + 1))
+        if [ "${count}" -lt "${retries}" ]; then
+            # There are still retries left, calculate delay and notify user.
+            backoff=$((2 * count))
+            if [[ "${backoff}" -gt "${MAX_BACKOFF}" ]]; then
+                backoff=${MAX_BACKOFF}
+            fi;
+
+            # Add a random amount to the delay to prevent competing processes
+            # from re-colliding.
+            wait=$(( backoff + (RANDOM % count) ))
+            echo "'${*}' Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            # Out of retries :(
+            echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}

tests-for-this-repo/helpers.bats

@@ -3,6 +3,17 @@
 
 . "${BASH_LIB_DIR}/init"
 
+# run before every test
+setup(){
+    temp_dir="${BATS_TMPDIR}/testtemp"
+    mkdir "${temp_dir}"
+    afile="${temp_dir}/appendfile"
+}
+
+teardown(){
+    temp_dir="${BATS_TMPDIR}/testtemp"
+    rm -rf "${temp_dir}"
+}
 
 @test "die exits and prints message" {
     run bash -c ". ${BASH_LIB_DIR}/init; die msg"
@@ -34,3 +45,70 @@
     assert_output --partial "stack empty"
     assert_failure
 }
+
+@test "retry runs command only once if it succeeds the first time" {
+    retryme(){
+        date >> ${afile}
+    }
+    run retry 3 retryme
+    assert_success
+    assert_equal $(wc -l <${afile}) 1
+}
+
+@test "retry doesn't introduce delay when the command succeeds first time" {
+    retryme(){
+        date >> ${afile}
+    }
+    start=$(date +%s)
+    run retry 3 retryme
+    end=$(date +%s)
+    assert [ "$(( start + 1 ))" -ge "${end}" ]
+    assert_success
+}
+
+@test "retry runs n times on consecutive failure and waits between attempts" {
+    retryme(){
+        date >> ${afile}
+        false
+    }
+    start=$(date +%s)
+    run retry 2 retryme
+    end=$(date +%s)
+    # introduces at least a two second delay between attempts
+    assert [ "$(( start + 2 ))" -le "${end}" ]
+    assert_failure
+    assert_equal $(wc -l <${afile}) 2
+}
+
+@test "retry returns after first success" {
+    retryme(){
+        date >> "${afile}"
+        case $(wc -l < ${afile}) in
+            *1)
+                return 1
+            ;;
+            *)
+                return 0
+            ;;
+        esac
+    }
+    run retry 3 retryme
+    assert_success
+    assert_equal $(wc -l <${afile}) 2
+}
+
+@test "retry fails with less than two arguments" {
+    run retry 3
+    assert_failure
+    assert_output --partial usage
+    assert [ ! -e "${temp_dir}/appendfile" ]
+}
+
+@test "retry fails with non-integer retry count" {
+    run retry "this" date
+    assert_failure
+    assert_output --partial number
+    assert [ ! -e "${temp_dir}/appendfile" ]
+}
+
+

tests-for-this-repo/k8s.bats

@@ -1,9 +1,7 @@
 . "${BASH_LIB_DIR}/test-utils/bats-support/load.bash"
 . "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash"
 
-setup(){
-    . "${BASH_LIB_DIR}/init"
-}
+. "${BASH_LIB_DIR}/init"
 
 @test "gke-utils image builds" {
     run build_gke_image

tests-for-this-repo/run-python-lint

@@ -5,7 +5,6 @@
 
 # shellcheck disable=SC2086,SC2046
 . $(dirname ${BASH_SOURCE[0]})/../init
-. "${BASH_LIB_DIR}/helpers/lib"
 
 rc=0
 
@@ -17,4 +16,4 @@ spushd ${BASH_LIB_DIR}/tests-for-this-repo/python-lint
     mv "${BASH_LIB_DIR}/junit.xml" "${BASH_LIB_DIR}/python-lint-junit.xml"
 spopd
 
-exit ${rc}
+exit ${rc}