Namespace Functions, Constant Retry & Logging (#17)

Namespace Functions, Constant Retry & Logging

Author: hughsaunders

CHANGES.md

@@ -0,0 +1,10 @@
+# Changelog
+
+## V2
+* V2.0.0
+  * Renamed all functions to include bl_ prefix
+  * Added bl_retry_constant
+  * Added bl_debug, bl_info, bl_warning, bl_error and bl_fatal logging functions
+
+## V1
+* v1.0.0: Initial Release

README.md

@@ -113,7 +113,7 @@ files within it's directory.
       <td>Functions relating to file and path handling
       <td>
         <ol>
-          <li> <b>abs_path</b>: Ensure a path is absolute</li>
+          <li><b>bl_abs_path</b>: Ensure a path is absolute</li>
         </ol>
       </td>
     </tr>
@@ -122,24 +122,27 @@ files within it's directory.
       <td>Git helpers</td>
       <td>
         <ol>
-          <li><b>repo_root</b>: Find the root of the current git repo.</li>
-          <li><b>all_files_in_repo</b>: List files tracked by git.</li>
-          <li><b>remote_latest_tag</b>: Returns the symbolic name of the latest tag from a remote.</li>
-          <li><b>remote_latest_tagged_commit</b>: Returns the SHA of the most recently tagged commit in a remote repo (<code>tag^{}</code>).</li>
-          <li><b>remote_sha_for_ref</b>: Returns the SHA for a given ref from a named remote.</li>
-          <li><b>remote_tag_for_sha</b>: Returns the tag corresponding to a SHA from a named remote - if there is one.</li>
-          <li><b>tracked_files_excluding_subtrees</b>: List files tracked by git, but excluding any files that are in paths listed in <code>.gittrees</code>.</li>
-          <li><b>cat_gittrees</b>: Returns the contents of .gittrees from the top level of the repo, excluding any comments. Fails if .gittrees is not present.</li>
+          <li><b>bl_repo_root</b>: Find the root of the current git repo.</li>
+          <li><b>bl_all_files_in_repo</b>: List files tracked by git.</li>
+          <li><b>bl_remote_latest_tag</b>: Returns the symbolic name of the latest tag from a remote.</li>
+          <li><b>bl_remote_latest_tagged_commit</b>: Returns the SHA of the most recently tagged commit in a remote repo (<code>tag^{}</code>).</li>
+          <li><b>bl_remote_sha_for_ref</b>: Returns the SHA for a given ref from a named remote.</li>
+          <li><b>bl_remote_tag_for_sha</b>: Returns the tag corresponding to a SHA from a named remote - if there is one.</li>
+          <li><b>bl_tracked_files_excluding_subtrees</b>: List files tracked by git, but excluding any files that are in paths listed in <code>.gittrees</code>.</li>
+          <li><b>bl_cat_gittrees</b>: Returns the contents of .gittrees from the top level of the repo, excluding any comments. Fails if .gittrees is not present.</li>
         </ol>
       </td>
     </tr>
       <td><a href="helpers/lib">helpers</a></td>
       <td>Bash scripting helpers</td>
       <td>
         <ol>
-          <li><b>die</b>: print message and exit 1</li>
-          <li><b>spushd/spopd</b>: Safe verisons of pushd & popd that call die if the push/pop fails, they also drop stdout. </li>
-          <li><b>retry</b>: Retry a command until it succeeds up to a user specified maximum number of attempts. Escalating delay between attempts.</li>
+          <li><b>bl_die</b>: print message and exit 1</li>
+          <li><b>bl_spushd/bl_spopd</b>: Safe verisons of pushd & popd that call die if the push/pop fails, they also drop stdout. </li>
+          <li><b>bl_is_num</b>: Check if a value is a number via regex</li>
+          <li><b>bl_retry</b>: Retry a command until it succeeds up to a user specified maximum number of attempts. Escalating delay between attempts.</li>
+          <li><b>bl_retry_constant</b>: Retry a command until it succeeds with a
+          constant delay between attempts</li>
         </ol>
       </td>
     </tr>
@@ -148,18 +151,26 @@ files within it's directory.
       <td>Utils for connecting to K8s</td>
       <td>
         <ol>
-          <li><b>build_gke_image</b>: Build docker image for running kubectl commands against GKE.</li>
-          <li><b>delete_gke_image</b>: Delete image from GKE.</li>
-          <li><b>run_docker_gke_command</b>: Run command in gke-utils container, already authenticated to k8s cluster.</li>
+          <li><b>bl_build_gke_image</b>: Build docker image for running kubectl commands against GKE.</li>
+          <li><b>bl_delete_gke_image</b>: Delete image from GKE.</li>
+          <li><b>bl_run_docker_gke_command</b>: Run command in gke-utils container, already authenticated to k8s cluster.</li>
         </ol>
       </td>
     </tr>
     <tr>
       <td><a href="logging/lib">logging</a></td>
-      <td>Helpers related to login</td>
+      <td>Helpers related to logging.</td>
       <td>
         <ol>
-          <li><b>announce</b>: Echo message in ascii banner to distinguish it from other log messages.</li>
+          <li><b>bl_announce</b>: Echo message in ascii banner to distinguish it from other log messages.</li>
+          <li><b>bl_log</b>: Log a message at the specified level. Default log level is info, change level by setting environment variable BASH_LIB_LOG_LEVEL</li>
+          <li><b>bl_check_log_level</b>: Check if a value is a valid bash lib
+          log level</li>
+          <li><b>bl_debug</b>: Log a message at debug level</li>
+          <li><b>bl_info</b>: Log a message at info level</li>
+          <li><b>bl_warning</b>: Log a message at warning level</li>
+          <li><b>bl_error</b>: Log a message at error level</li>
+          <li><b>bl_fatal</b>: Log a message at fatal level</li>
         </ol>
       </td>
     </tr>
@@ -168,9 +179,9 @@ files within it's directory.
       <td>Helpers for executing tests</td>
       <td>
         <ol>
-          <li><b>shellcheck_script</b>: Execute shellcheck against a script, uses docker.</li>
-          <li><b>find_scripts</b>: Find git tracked files with extension.</li>
-          <li><b>tap2junit</b>: Convert a subset of <a href="http://testanything.org/">TAP</a> to JUnit XML. Retains logs for errors.</li>
+          <li><b>bl_shellcheck_script</b>: Execute shellcheck against a script, uses docker.</li>
+          <li><b>bl_find_scripts</b>: Find git tracked files with extension.</li>
+          <li><b>bl_tap2junit</b>: Convert a subset of <a href="http://testanything.org/">TAP</a> to JUnit XML. Retains logs for errors.</li>
         </ol>
       </td>
     </tr>

filehandling/lib

@@ -3,7 +3,7 @@
 : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
 
 #https://stackoverflow.com/a/23002317
-function abs_path() {
+function bl_abs_path() {
     # generate absolute path from relative path
     # path     : relative filename
     # return : absolute path
@@ -14,13 +14,13 @@ function abs_path() {
     fi
     if [ -d "${path}" ]; then
         # dir
-        (spushd "${path}"; pwd)
+        (bl_spushd "${path}"; pwd)
     elif [ -f "${path}" ]; then
         # file
         if [[ ${path} = /* ]]; then
             echo "${path}"
         elif [[ ${path} == */* ]]; then
-            echo "$(spushd "${path%/*}"; pwd)/${path##*/}"
+            echo "$(bl_spushd "${path%/*}"; pwd)/${path##*/}"
         else
             echo "$(pwd)/${path}"
         fi

git/lib

@@ -3,18 +3,18 @@
 : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
 
 # Get the top level of a git repo
-function repo_root(){
+function bl_repo_root(){
     git rev-parse --show-toplevel
 }
 
 # List files tracked by git
-function all_files_in_repo(){
+function bl_all_files_in_repo(){
     git ls-tree -r HEAD --name-only
 }
 
 # Find the latest tag available at a repo url
 # Returns tag name, not sha
-function remote_latest_tag(){
+function bl_remote_latest_tag(){
     local -r remote_url="${1}"
     # In ls-remote the ^{} suffix refers to a peeled/dereferenced object.
     # eg refs/tags/v0.0.1^{} shows the SHA of the commit that was tagged,
@@ -28,13 +28,13 @@ function remote_latest_tag(){
 }
 
 # Find the SHA of the latests commit to be tagged in a remote repo
-function remote_latest_tagged_commit(){
+function bl_remote_latest_tagged_commit(){
     local -r remote="${1}"
-    local -r tag="$(remote_latest_tag "${remote}")"
+    local -r tag="$(bl_remote_latest_tag "${remote}")"
     git ls-remote "${remote}" | awk "/refs\/tags\/${tag}\^/{print \$1}"
 }
 
-function remote_sha_for_ref(){
+function bl_remote_sha_for_ref(){
     local -r remote="${1}"
     local -r ref="${2}"
 
@@ -54,7 +54,7 @@ function remote_sha_for_ref(){
     fi
 }
 
-function remote_tag_for_sha(){
+function bl_remote_tag_for_sha(){
     local -r remote="${1}"
     local -r sha="${2}"
     git ls-remote "${remote}" \
@@ -64,15 +64,15 @@ function remote_tag_for_sha(){
 
 ## Minimal git subtree functionality required for tests to pass
 # full subtree functionality is not ready for merge.
-function cat_gittrees(){
-    local -r git_trees="$(repo_root)/.gittrees"
+function bl_cat_gittrees(){
+    local -r git_trees="$(bl_repo_root)/.gittrees"
     local -r subtrees_file_format=".gittrees should contain one subtree per line,\
 space seperated with three fields: subtree_path renmote_url remote_name"
-    [[ -e "${git_trees}" ]] || die ".gittrees file ${git_trees} not found. ${subtrees_file_format}"
-    grep -E -v '^\s*$|^\s*#' "$(repo_root)/.gittrees"
+    [[ -e "${git_trees}" ]] || bl_die ".gittrees file ${git_trees} not found. ${subtrees_file_format}"
+    grep -E -v '^\s*$|^\s*#' "$(bl_repo_root)/.gittrees"
 }
 
-function tracked_files_excluding_subtrees(){
-    subtrees="$(cat_gittrees | awk '{print $1}' | paste -sd '|' -)"
-    all_files_in_repo | grep -E -v "${subtrees}"
+function bl_tracked_files_excluding_subtrees(){
+    subtrees="$(bl_cat_gittrees | awk '{print $1}' | paste -sd '|' -)"
+    bl_all_files_in_repo | grep -E -v "${subtrees}"
 }

helpers/lib

@@ -2,32 +2,37 @@
 
 : "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}"
 
-function die(){
+function bl_die(){
     echo "${@}"
     exit 1
 }
 
 #safe pushd
-function spushd(){
+function bl_spushd(){
     if ! pushd "${1}" >/dev/null; then
         die "pushd ${1} failed :("
     fi
 }
 
 #safe popd
-function spopd(){
-    popd >/dev/null || die "popd failed :("
+function bl_spopd(){
+    popd >/dev/null || bl_die "popd failed :("
+}
+
+# Test if a variable contains a number
+function bl_is_num(){
+    [[ ${1:-invalid} =~ ^-?[0-9\.]*$ ]]
 }
 
 # Retry a command multiple times until it succeeds, with escalating
 # delay between attempts.
 # Delay is 2 * n + random up to 30s, then 30s + random after that.
 # For large numbers of retries the max delay is effectively the retry
-# in minutes.
+# count in minutes.
 # Based on:
 # https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
 # but now quite heavily modified.
-function retry {
+function bl_retry {
     # Maxiumum amount of fixed delay between attempts
     # a random value will still be added.
     local -r MAX_BACKOFF=30
@@ -40,7 +45,7 @@ function retry {
     local retries=$1
     shift
 
-    if ! [[ ${retries} =~ ^[0-9\.]*$ ]]; then
+    if ! bl_is_num "${retries}"; then
         echo "Invalid number of retries: ${retries} for command '${*}'".
         exit 1
     fi
@@ -72,3 +77,42 @@ function retry {
     done
     return 0
 }
+
+# retry function that waits a constant number of seconds between attempts.
+function bl_retry_constant {
+    if [[ ${#} -lt 3 ]]; then
+        echo "retry usage: retry <retries> <interval (seconds)> <command>"
+        exit 1
+    fi
+
+    local retries=$1; shift
+    local interval=$1; shift
+
+    if ! bl_is_num "${retries}"; then
+        echo "Invalid number of retries: ${retries} for command '${*}'".
+        exit 1
+    fi
+
+    if ! bl_is_num "${interval}"; then
+        echo "Invalid interval in seconds: ${retries} for command '${*}'".
+        exit 1
+    fi
+
+    local count=0
+    until eval "$@"; do
+        # Command failed, otherwise until would have skipped the loop
+
+        # Store return code so it can be reported to the user
+        exit=$?
+        count=$((count + 1))
+        if [ "${count}" -lt "${retries}" ]; then
+            echo "'${*}' Retry $count/$retries exited $exit, retrying in $interval seconds..."
+            sleep "${interval}"
+        else
+            # Out of retries :(
+            echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}

init

@@ -1,9 +1,15 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 ## Initialisation Functions for the
 ## Conjurinc Bash Library
 
-# Shell Options
+if (( BASH_VERSINFO[0] < 4 )); then
+    echo "Bash Lib requires bash v4 or greater"
+    echo "Current Bash Version: ${BASH_VERSION}"
+    exit 1
+fi
+
+# Shell Otions
 set -euo pipefail
 
 # This script should be sourced before any of
@@ -24,17 +30,17 @@ for lib in helpers logging filehandling git k8s test-utils; do
     . "${BASH_LIB_DIR_RELATIVE}/${lib}/lib"
 done
 
-# Export functions to subshells
-eval "$(declare -F | sed -e 's/-f /-fx /')"
+# Filter functions and re export only bash-lib functions to subshells
+eval "$(declare -F | sed -e 's/-f /-fx /' | grep 'x bl_')"
 
 # Export the absolute path
 # shellcheck disable=SC2086
-BASH_LIB_DIR="$(abs_path ${BASH_LIB_DIR_RELATIVE})"
+BASH_LIB_DIR="$(bl_abs_path ${BASH_LIB_DIR_RELATIVE})"
 export BASH_LIB_DIR
 
 # Update Submodules
-spushd "${BASH_LIB_DIR}"
+bl_spushd "${BASH_LIB_DIR}"
     git submodule update --init --recursive
-spopd
+bl_spopd
 
 export BATS_CMD="${BASH_LIB_DIR}/test-utils/bats/bin/bats"

k8s/lib

@@ -4,33 +4,33 @@
 
 # Sets additional required environment variables that aren't available in the
 # secrets.yml file, and performs other preparatory steps
-function build_gke_image() {
+function bl_build_gke_image() {
   local image="gke-utils:latest"
   local rc=0
   docker rmi ${image} || true
-  spushd "${BASH_LIB_DIR}/k8s"
+  bl_spushd "${BASH_LIB_DIR}/k8s"
     # Prepare Docker images
     docker build --tag "${image}"\
       --build-arg KUBECTL_CLI_URL="${KUBECTL_CLI_URL}" \
       . 1>&2
     rc=${?}
-  spopd
+  bl_spopd
 
   return ${rc}
 }
 
 # Delete an image from GCR, unless it is has multiple tags pointing to it
 # This means another parallel build is using the image and we should
 # just untag it to be deleted by the later job
-function delete_gke_image() {
+function bl_delete_gke_image() {
   local image_and_tag="${1}"
 
-  run_docker_gke_command "
+  bl_run_docker_gke_command "
     gcloud container images delete --force-delete-tags -q ${image_and_tag}
   "
 }
 
-function run_docker_gke_command() {
+function bl_run_docker_gke_command() {
   docker run --rm \
     -i \
     -e DOCKER_REGISTRY_URL \