Merge commit 'c480574b3ac7ba83679ea1fba6739c1825093c1f' into nodeport-upload

Author: mccheah

resource-managers/kubernetes/README.md

@@ -23,7 +23,7 @@ To build a distribution of Spark with Kubernetes support, use the `dev/make-dist
 `kubernetes` profile as part of the build arguments. Any other build arguments can be specified as one would expect when
 building Spark normally. For example, to build Spark against Hadoop 2.7 and Kubernetes:
 
-    dev/make-distribution.sh --tgz -Phadoop2.7 -Pkubernetes
+    dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes
 
 # Kubernetes Code Modules
 

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/Client.scala

@@ -58,6 +58,8 @@ private[spark] class Client(
     "spark.kubernetes.driver.docker.image", s"spark-driver:$SPARK_VERSION")
   private val uploadedJars = sparkConf.getOption("spark.kubernetes.driver.uploads.jars")
   private val uiPort = sparkConf.getInt("spark.ui.port", DEFAULT_UI_PORT)
+  private val driverLaunchTimeoutSecs = sparkConf.getTimeAsSeconds(
+    "spark.kubernetes.driverLaunchTimeout", s"${DEFAULT_LAUNCH_TIMEOUT_SECONDS}s")
 
   private val secretBase64String = {
     val secretBytes = new Array[Byte](128)
@@ -254,7 +256,7 @@ private[spark] class Client(
             .done()
           var submitSucceeded = false
           try {
-            submitCompletedFuture.get(LAUNCH_TIMEOUT_SECONDS, TimeUnit.SECONDS)
+            submitCompletedFuture.get(driverLaunchTimeoutSecs, TimeUnit.SECONDS)
             submitSucceeded = true
           } catch {
             case e: TimeoutException =>
@@ -290,10 +292,10 @@ private[spark] class Client(
       kubernetesClient.pods().withName(kubernetesAppId).get()
     } catch {
       case throwable: Throwable =>
-        logError(s"Timed out while waiting $LAUNCH_TIMEOUT_SECONDS seconds for the" +
+        logError(s"Timed out while waiting $driverLaunchTimeoutSecs seconds for the" +
           " driver pod to start, but an error occurred while fetching the driver" +
           " pod's details.", throwable)
-        throw new SparkException(s"Timed out while waiting $LAUNCH_TIMEOUT_SECONDS" +
+        throw new SparkException(s"Timed out while waiting $driverLaunchTimeoutSecs" +
           " seconds for the driver pod to start. Unfortunately, in attempting to fetch" +
           " the latest state of the pod, another error was thrown. Check the logs for" +
           " the error that was thrown in looking up the driver pod.", e)
@@ -440,7 +442,7 @@ private[spark] object Client extends Logging {
   private val DRIVER_LAUNCHER_CONTAINER_NAME = "spark-kubernetes-driver-launcher"
   private val SECURE_RANDOM = new SecureRandom()
   private val SPARK_SUBMISSION_SECRET_BASE_DIR = "/var/run/secrets/spark-submission"
-  private val LAUNCH_TIMEOUT_SECONDS = 30
+  private val DEFAULT_LAUNCH_TIMEOUT_SECONDS = 60
   private val SPARK_APP_NAME_LABEL = "spark-app-name"
 
   def main(args: Array[String]): Unit = {