:run: cabpk: Add retries to control plane join

Signed-off-by: Naadir Jeewa <[email protected]>

Author: Naadir Jeewa

bootstrap/kubeadm/internal/cloudinit/controlplane_join.go

@@ -18,11 +18,136 @@ package cloudinit
 
 import (
 	"github.com/pkg/errors"
-
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/util/secret"
 )
 
 const (
+	controlPlaneJoinScript = `#!/bin/bash
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Log an error and exit.
+# Args:
+#   $1 Message to log with the error
+#   $2 The error code to return
+log::error_exit() {
+  local message="${1}"
+  local code="${2}"
+
+  log::error "${message}"
+  log::info "Removing member from cluster status"
+  kubeadm reset -f update-cluster-status || true
+  log::info "Removing etcd member"
+  kubeadm reset -f remove-etcd-member || true
+  log::info "Resetting kubeadm"
+  kubeadm reset -f || true
+  log::error "cluster.x-k8s.io kubeadm bootstrap script $0 exiting with status ${code}"
+  exit "${code}"
+}
+
+log::success_exit() {
+  log::info "cluster.x-k8s.io kubeadm bootstrap script $0 finished"
+  exit 0
+}
+
+# Log an error but keep going.
+log::error() {
+  local message="${1}"
+  timestamp=$(date --iso-8601=seconds)
+  echo "!!! [${timestamp}] ${1}" >&2
+  shift
+  for message; do
+    echo "    ${message}" >&2
+  done
+}
+
+# Print a status line.  Formatted to show up in a stream of output.
+log::info() {
+  timestamp=$(date --iso-8601=seconds)
+  echo "+++ [${timestamp}] ${1}"
+  shift
+  for message; do
+    echo "    ${message}"
+  done
+}
+
+check_kubeadm_command() {
+  local command="${1}"
+  local code="${2}"
+  local out="${3}"
+  local sanitised="${out//[$'\t\r\n']/}"
+  case ${code} in
+  "0")
+    log::info "kubeadm reported successful execution for ${command}"
+    ;;
+  "1")
+    log::error "kubeadm reported failed action(s) for ${command}"
+    log::error "${sanitised}"
+    ;;
+  "2")
+    log::error "kubeadm reported preflight check error during ${command}"
+    log::error "${sanitised}"
+    ;;
+  "3")
+    log::error "kubeadm reported validation error for ${command}"
+    log::error_exit "${sanitised}"
+    ;;
+  *)
+    log::error "kubeadm reported unknown error ${code} for ${command}"
+    log::error "${sanitised}"
+    ;;
+  esac
+}
+
+function retry-command() {
+  n=0
+  local kubeadm_return
+  until [ $n -ge 5 ]; do
+    local out
+    out=$(
+      "$@" --config /tmp/kubeadm-controlplane-join-config.yaml {{.KubeadmVerbosity}}
+    )
+    kubeadm_return=$?
+    check_kubeadm_command "'$*'" "${kubeadm_return}" "${out}"
+    if [ ${kubeadm_return} -eq 0 ]; then
+      break
+    fi
+    # We allow preflight errors to pass
+    if [ ${kubeadm_return} -eq 2 ]; then
+      break
+    fi
+    n=$((n + 1))
+    log::info
+    sleep 15
+  done
+  if [ ${kubeadm_return} -ne 0 ]; then
+    log::error_exit "too many errors, exiting"
+  fi
+}
+
+retry-command kubeadm join phase preflight
+retry-command kubeadm join phase control-plane-prepare download-certs
+retry-command kubeadm join phase control-plane-prepare certs
+retry-command kubeadm join phase control-plane-prepare kubeconfig
+retry-command kubeadm join phase kubelet-start
+retry-command kubeadm join phase control-plane-join etcd
+retry-command kubeadm join phase control-plane-join update-status
+retry-command kubeadm join phase control-plane-join mark-control-plane
+
+log::success_exit
+`
 	controlPlaneJoinCloudInit = `{{.Header}}
 {{template "files" .WriteFiles}}
 -   path: /tmp/kubeadm-controlplane-join-config.yaml
@@ -32,7 +157,7 @@ const (
 {{.JoinConfiguration | Indent 6}}
 runcmd:
 {{- template "commands" .PreKubeadmCommands }}
-  - 'kubeadm join --config /tmp/kubeadm-controlplane-join-config.yaml {{.KubeadmVerbosity}}'
+  - '/usr/local/bin/kubeadm-bootstrap-script'
 {{- template "commands" .PostKubeadmCommands }}
 {{- template "ntp" .NTP }}
 {{- template "users" .Users }}
@@ -54,10 +179,21 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) ([]byte, error) {
 	// TODO: Consider validating that the correct certificates exist. It is different for external/stacked etcd
 	input.WriteFiles = input.Certificates.AsFiles()
 	input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...)
+	joinScript, err := generate("JoinControlplaneScript", controlPlaneJoinScript, input)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to generate user data for machine joining control plane")
+	}
+	joinScriptFile := bootstrapv1.File{
+		Path:        "/usr/local/bin/kubeadm-bootstrap-script",
+		Owner:       "root",
+		Permissions: "0755",
+		Content:     string(joinScript),
+	}
+	input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...)
+	input.WriteFiles = append(input.WriteFiles, joinScriptFile)
 	userData, err := generate("JoinControlplane", controlPlaneJoinCloudInit, input)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to generate user data for machine joining control plane")
 	}
-
 	return userData, err
 }

feature/feature.go

@@ -31,6 +31,10 @@ const (
 	// owner: @
 	// alpha: v0.3
 	MachinePool featuregate.Feature = "MachinePool"
+
+	// owner: @randomvariable
+	// alpha: v0.3
+	RetriableBootstrap featuregate.Feature = "RetriableBootstrap"
 )
 
 func init() {
@@ -42,4 +46,5 @@ func init() {
 var defaultClusterAPIFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
 	// Every feature should be initiated here:
 	MachinePool: {Default: false, PreRelease: featuregate.Alpha},
+	RetriableBootstrap: {Default: false, PreRelease: featuregate.Alpha}
 }