[7.10][ML] Retry pulling of Docker images to work around network issues (#1542)

Adds a function to retry pulling Docker images up to 5 times, and
calls it to pre-fetch every Docker image that's used as a "FROM"
image by our "docker build" commands.

"docker build" will pull the "FROM" image if it's not present on
the local machine, but will only try once, so is vulnerable to a
transient problem with the Docker registry failing the whole build.
By retrying "docker pull" several times before running "docker build"
we ensure that the image is present locally when "docker build" runs
and hence "docker build" never needs to pull it.

Backport of #1535

Author: David Roberts

dev-tools/docker/build_check_style_image.sh

@@ -21,9 +21,12 @@ set -e
 
 cd `dirname $0`
 
-docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION check_style_image
+. ./prefetch_docker_image.sh
+CONTEXT=check_style_image
+prefetch_docker_image $CONTEXT/Dockerfile
+docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT
 # Get a username and password for this by visiting
-# https://docker.elastic.co:7000 and allowing it to authenticate against your
+# https://docker-auth.elastic.co and allowing it to authenticate against your
 # GitHub account
 docker login $HOST
 docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION

dev-tools/docker/build_linux_aarch64_cross_build_image.sh

@@ -23,9 +23,12 @@ set -e
 
 cd `dirname $0`
 
-docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_aarch64_cross_image
+. ./prefetch_docker_image.sh
+CONTEXT=linux_aarch64_cross_image
+prefetch_docker_image $CONTEXT/Dockerfile
+docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT
 # Get a username and password for this by visiting
-# https://docker.elastic.co:7000 and allowing it to authenticate against your
+# https://docker-auth.elastic.co and allowing it to authenticate against your
 # GitHub account
 docker login $HOST
 docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION

dev-tools/docker/build_linux_aarch64_native_build_image.sh

@@ -29,9 +29,12 @@ set -e
 
 cd `dirname $0`
 
-docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_aarch64_native_image
+. ./prefetch_docker_image.sh
+CONTEXT=linux_aarch64_native_image
+prefetch_docker_image $CONTEXT/Dockerfile
+docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT
 # Get a username and password for this by visiting
-# https://docker.elastic.co:7000 and allowing it to authenticate against your
+# https://docker-auth.elastic.co and allowing it to authenticate against your
 # GitHub account
 docker login $HOST
 docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION

dev-tools/docker/build_linux_build_image.sh

@@ -29,9 +29,12 @@ set -e
 
 cd `dirname $0`
 
-docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_image
+. ./prefetch_docker_image.sh
+CONTEXT=linux_image
+prefetch_docker_image $CONTEXT/Dockerfile
+docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT
 # Get a username and password for this by visiting
-# https://docker.elastic.co:7000 and allowing it to authenticate against your
+# https://docker-auth.elastic.co and allowing it to authenticate against your
 # GitHub account
 docker login $HOST
 docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION

dev-tools/docker/build_macosx_build_image.sh

@@ -23,9 +23,12 @@ set -e
 
 cd `dirname $0`
 
-docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION macosx_image
+. ./prefetch_docker_image.sh
+CONTEXT=macosx_image
+prefetch_docker_image $CONTEXT/Dockerfile
+docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT
 # Get a username and password for this by visiting
-# https://docker.elastic.co:7000 and allowing it to authenticate against your
+# https://docker-auth.elastic.co and allowing it to authenticate against your
 # GitHub account
 docker login $HOST
 docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION

dev-tools/docker/prefetch_docker_image.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# Pull a docker image, retrying up to 5 times.
+#
+# Making sure the "FROM" image is present locally before building an image
+# based on it removes the risk of a "docker build" failing due to transient
+# Docker registry problems.
+#
+# The argument is the path to the Dockerfile to be built.
+function prefetch_docker_image {
+    DOCKERFILE="$1"
+    IMAGE=$(grep '^FROM' "$DOCKERFILE" | awk '{ print $2 }' | head -1)
+    ATTEMPT=0
+    MAX_RETRIES=5
+  
+    while true
+    do
+        ATTEMPT=$((ATTEMPT+1))
+  
+        if [ $ATTEMPT -gt $MAX_RETRIES ] ; then
+            echo "Docker pull retries exceeded, aborting."
+            exit 1
+        fi
+  
+        if docker pull "$IMAGE" ; then
+            echo "Docker pull of $IMAGE successful."
+            break
+        else
+            echo "Docker pull of $IMAGE unsuccessful, attempt '$ATTEMPT'."
+        fi
+    done
+}
+

dev-tools/docker_build.sh

@@ -62,6 +62,8 @@ cd "$TOOLS_DIR/.."
 # necessary network access
 3rd_party/pull-eigen.sh
 
+. "$TOOLS_DIR/docker/prefetch_docker_image.sh"
+
 for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u`
 do
 
@@ -72,6 +74,7 @@ do
     DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_builder/Dockerfile"
     TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$
 
+    prefetch_docker_image "$DOCKERFILE"
     docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT -f "$DOCKERFILE" .
     # Using tar to copy the build artifacts out of the container seems more reliable
     # than docker cp, and also means the files end up with the correct uid/gid

dev-tools/docker_check_style.sh

@@ -26,6 +26,8 @@ cd "$TOOLS_DIR/.."
 DOCKERFILE="$TOOLS_DIR/docker/style_checker/Dockerfile"
 TEMP_TAG=`git rev-parse --short=14 HEAD`-style-$$
 
+. "$TOOLS_DIR/docker/prefetch_docker_image.sh"
+prefetch_docker_image "$DOCKERFILE"
 docker build --no-cache --force-rm -t $TEMP_TAG -f "$DOCKERFILE" .
 docker run --rm --workdir=/ml-cpp $TEMP_TAG dev-tools/check-style.sh --all
 RC=$?

dev-tools/docker_test.sh

@@ -62,6 +62,8 @@ cd "$TOOLS_DIR/.."
 # necessary network access
 3rd_party/pull-eigen.sh
 
+. "$TOOLS_DIR/docker/prefetch_docker_image.sh"
+
 for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u`
 do
 
@@ -73,6 +75,7 @@ do
     DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_tester/Dockerfile"
     TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$
 
+    prefetch_docker_image "$DOCKERFILE"
     docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT -f "$DOCKERFILE" .
     # Using tar to copy the build and test artifacts out of the container seems
     # more reliable than docker cp, and also means the files end up with the

dev-tools/jenkins_ci.sh

@@ -67,6 +67,9 @@ git repack -a -d
 readonly GIT_TOPLEVEL=$(git rev-parse --show-toplevel 2> /dev/null)
 rm -f "${GIT_TOPLEVEL}/.git/objects/info/alternates"
 
+# The Docker version is helpful to identify version-specific Docker bugs
+docker --version
+
 # Build and test the native Linux architecture
 if [ "$HARDWARE_ARCH" = x86_64 ] ; then