From 890a0bb4e0df742222a3055b86c3f709edbb97cb Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 15 Oct 2020 15:37:50 +0100 Subject: [PATCH] [7.10][ML] Retry pulling of Docker images to work around network issues Adds a function to retry pulling Docker images up to 5 times, and calls it to pre-fetch every Docker image that's used as a "FROM" image by our "docker build" commands. "docker build" will pull the "FROM" image if it's not present on the local machine, but will only try once, so is vulnerable to a transient problem with the Docker registry failing the whole build. By retrying "docker pull" several times before running "docker build" we ensure that the image is present locally when "docker build" runs and hence "docker build" never needs to pull it. Backport of #1535 --- dev-tools/docker/build_check_style_image.sh | 7 +++- .../build_linux_aarch64_cross_build_image.sh | 7 +++- .../build_linux_aarch64_native_build_image.sh | 7 +++- dev-tools/docker/build_linux_build_image.sh | 7 +++- dev-tools/docker/build_macosx_build_image.sh | 7 +++- dev-tools/docker/prefetch_docker_image.sh | 38 +++++++++++++++++++ dev-tools/docker_build.sh | 3 ++ dev-tools/docker_check_style.sh | 2 + dev-tools/docker_test.sh | 3 ++ dev-tools/jenkins_ci.sh | 3 ++ 10 files changed, 74 insertions(+), 10 deletions(-) create mode 100755 dev-tools/docker/prefetch_docker_image.sh diff --git a/dev-tools/docker/build_check_style_image.sh b/dev-tools/docker/build_check_style_image.sh index 257d166f11..0f483315f2 100755 --- a/dev-tools/docker/build_check_style_image.sh +++ b/dev-tools/docker/build_check_style_image.sh @@ -21,9 +21,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION check_style_image +. ./prefetch_docker_image.sh +CONTEXT=check_style_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/build_linux_aarch64_cross_build_image.sh b/dev-tools/docker/build_linux_aarch64_cross_build_image.sh index f8961872df..956820d518 100755 --- a/dev-tools/docker/build_linux_aarch64_cross_build_image.sh +++ b/dev-tools/docker/build_linux_aarch64_cross_build_image.sh @@ -23,9 +23,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_aarch64_cross_image +. ./prefetch_docker_image.sh +CONTEXT=linux_aarch64_cross_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/build_linux_aarch64_native_build_image.sh b/dev-tools/docker/build_linux_aarch64_native_build_image.sh index f5e35b5bf0..4621751ef9 100755 --- a/dev-tools/docker/build_linux_aarch64_native_build_image.sh +++ b/dev-tools/docker/build_linux_aarch64_native_build_image.sh @@ -29,9 +29,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_aarch64_native_image +. ./prefetch_docker_image.sh +CONTEXT=linux_aarch64_native_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/build_linux_build_image.sh b/dev-tools/docker/build_linux_build_image.sh index ae9f68bdfa..8d19bca9f9 100755 --- a/dev-tools/docker/build_linux_build_image.sh +++ b/dev-tools/docker/build_linux_build_image.sh @@ -29,9 +29,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_image +. ./prefetch_docker_image.sh +CONTEXT=linux_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/build_macosx_build_image.sh b/dev-tools/docker/build_macosx_build_image.sh index a52fd3123b..c82e40e835 100755 --- a/dev-tools/docker/build_macosx_build_image.sh +++ b/dev-tools/docker/build_macosx_build_image.sh @@ -23,9 +23,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION macosx_image +. ./prefetch_docker_image.sh +CONTEXT=macosx_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/prefetch_docker_image.sh b/dev-tools/docker/prefetch_docker_image.sh new file mode 100755 index 0000000000..aa315104a0 --- /dev/null +++ b/dev-tools/docker/prefetch_docker_image.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Pull a docker image, retrying up to 5 times. +# +# Making sure the "FROM" image is present locally before building an image +# based on it removes the risk of a "docker build" failing due to transient +# Docker registry problems. +# +# The argument is the path to the Dockerfile to be built. +function prefetch_docker_image { + DOCKERFILE="$1" + IMAGE=$(grep '^FROM' "$DOCKERFILE" | awk '{ print $2 }' | head -1) + ATTEMPT=0 + MAX_RETRIES=5 + + while true + do + ATTEMPT=$((ATTEMPT+1)) + + if [ $ATTEMPT -gt $MAX_RETRIES ] ; then + echo "Docker pull retries exceeded, aborting." + exit 1 + fi + + if docker pull "$IMAGE" ; then + echo "Docker pull of $IMAGE successful." + break + else + echo "Docker pull of $IMAGE unsuccessful, attempt '$ATTEMPT'." + fi + done +} + diff --git a/dev-tools/docker_build.sh b/dev-tools/docker_build.sh index c610a1dd86..1b1c2b625e 100755 --- a/dev-tools/docker_build.sh +++ b/dev-tools/docker_build.sh @@ -62,6 +62,8 @@ cd "$TOOLS_DIR/.." # necessary network access 3rd_party/pull-eigen.sh +. "$TOOLS_DIR/docker/prefetch_docker_image.sh" + for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u` do @@ -72,6 +74,7 @@ do DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_builder/Dockerfile" TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$ + prefetch_docker_image "$DOCKERFILE" docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT -f "$DOCKERFILE" . # Using tar to copy the build artifacts out of the container seems more reliable # than docker cp, and also means the files end up with the correct uid/gid diff --git a/dev-tools/docker_check_style.sh b/dev-tools/docker_check_style.sh index 0ee2a8f7a7..f3f2419ca2 100755 --- a/dev-tools/docker_check_style.sh +++ b/dev-tools/docker_check_style.sh @@ -26,6 +26,8 @@ cd "$TOOLS_DIR/.." DOCKERFILE="$TOOLS_DIR/docker/style_checker/Dockerfile" TEMP_TAG=`git rev-parse --short=14 HEAD`-style-$$ +. "$TOOLS_DIR/docker/prefetch_docker_image.sh" +prefetch_docker_image "$DOCKERFILE" docker build --no-cache --force-rm -t $TEMP_TAG -f "$DOCKERFILE" . docker run --rm --workdir=/ml-cpp $TEMP_TAG dev-tools/check-style.sh --all RC=$? diff --git a/dev-tools/docker_test.sh b/dev-tools/docker_test.sh index f4df21c709..868258e5fd 100755 --- a/dev-tools/docker_test.sh +++ b/dev-tools/docker_test.sh @@ -62,6 +62,8 @@ cd "$TOOLS_DIR/.." # necessary network access 3rd_party/pull-eigen.sh +. "$TOOLS_DIR/docker/prefetch_docker_image.sh" + for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u` do @@ -73,6 +75,7 @@ do DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_tester/Dockerfile" TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$ + prefetch_docker_image "$DOCKERFILE" docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT -f "$DOCKERFILE" . # Using tar to copy the build and test artifacts out of the container seems # more reliable than docker cp, and also means the files end up with the diff --git a/dev-tools/jenkins_ci.sh b/dev-tools/jenkins_ci.sh index 3b280157be..9da20ace1f 100755 --- a/dev-tools/jenkins_ci.sh +++ b/dev-tools/jenkins_ci.sh @@ -67,6 +67,9 @@ git repack -a -d readonly GIT_TOPLEVEL=$(git rev-parse --show-toplevel 2> /dev/null) rm -f "${GIT_TOPLEVEL}/.git/objects/info/alternates" +# The Docker version is helpful to identify version-specific Docker bugs +docker --version + # Build and test the native Linux architecture if [ "$HARDWARE_ARCH" = x86_64 ] ; then