From c925317c12b55b38b3ca8b4d561788f53e82aacf Mon Sep 17 00:00:00 2001 From: foxish Date: Mon, 6 Nov 2017 00:30:12 -0800 Subject: [PATCH 1/4] Spark Submit Unit tests --- .../org/apache/spark/deploy/SparkSubmit.scala | 10 ++++---- .../spark/deploy/SparkSubmitSuite.scala | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 85b76013ba5f3..38b57c1a01279 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -483,20 +483,20 @@ object SparkSubmit extends CommandLineUtils { sysProp = "spark.kubernetes.namespace"), // Other options - OptionAssigner(args.executorCores, STANDALONE | YARN, ALL_DEPLOY_MODES, + OptionAssigner(args.executorCores, STANDALONE | YARN | KUBERNETES, ALL_DEPLOY_MODES, sysProp = "spark.executor.cores"), - OptionAssigner(args.executorMemory, STANDALONE | MESOS | YARN, ALL_DEPLOY_MODES, + OptionAssigner(args.executorMemory, STANDALONE | MESOS | YARN | KUBERNETES, ALL_DEPLOY_MODES, sysProp = "spark.executor.memory"), - OptionAssigner(args.totalExecutorCores, STANDALONE | MESOS, ALL_DEPLOY_MODES, + OptionAssigner(args.totalExecutorCores, STANDALONE | MESOS | KUBERNETES, ALL_DEPLOY_MODES, sysProp = "spark.cores.max"), OptionAssigner(args.files, LOCAL | STANDALONE | MESOS | KUBERNETES, ALL_DEPLOY_MODES, sysProp = "spark.files"), OptionAssigner(args.jars, LOCAL, CLIENT, sysProp = "spark.jars"), OptionAssigner(args.jars, STANDALONE | MESOS | KUBERNETES, ALL_DEPLOY_MODES, sysProp = "spark.jars"), - OptionAssigner(args.driverMemory, STANDALONE | MESOS | YARN, CLUSTER, + OptionAssigner(args.driverMemory, STANDALONE | MESOS | YARN | KUBERNETES, CLUSTER, sysProp = "spark.driver.memory"), - OptionAssigner(args.driverCores, STANDALONE | MESOS | YARN, CLUSTER, + OptionAssigner(args.driverCores, STANDALONE | MESOS | YARN | KUBERNETES, CLUSTER, sysProp = "spark.driver.cores"), OptionAssigner(args.supervise.toString, STANDALONE | MESOS, CLUSTER, sysProp = "spark.driver.supervise"), diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 6fa3a09b2ef1e..beb723b8420be 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -381,6 +381,29 @@ class SparkSubmitSuite sysProps("spark.ui.enabled") should be ("false") } + test("handles k8s cluster mode") { + val clArgs = Seq( + "--deploy-mode", "cluster", + "--master", "k8s://h:p", + "--executor-memory", "5g", + "--class", "org.SomeClass", + "--kubernetes-namespace", "foo", + "--driver-memory", "4g", + "thejar.jar", + "arg1") + val appArgs = new SparkSubmitArguments(clArgs) + val (childArgs, classpath, sysProps, mainClass) = prepareSubmitEnvironment(appArgs) + val childArgsMap = childArgs.grouped(2).map(a => a(0) -> a(1)).toMap + assert(childArgsMap.contains("--primary-java-resource")) + assert(childArgsMap.get("--main-class") === Some("org.SomeClass")) + assert(childArgsMap.get("--arg") === Some("arg1")) + + mainClass should be ("org.apache.spark.deploy.k8s.submit.Client") + classpath should have length (0) + sysProps("spark.executor.memory") should be ("5g") + sysProps("spark.kubernetes.namespace") should be ("foo") + } + test("handles confs with flag equivalents") { val clArgs = Seq( "--deploy-mode", "cluster", From 9b7d2bdf65f14898d81edf40bd8a896cecfd60f1 Mon Sep 17 00:00:00 2001 From: foxish Date: Mon, 6 Nov 2017 00:32:03 -0800 Subject: [PATCH 2/4] Improvements --- .../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index beb723b8420be..e53d608ff192c 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -394,9 +394,9 @@ class SparkSubmitSuite val appArgs = new SparkSubmitArguments(clArgs) val (childArgs, classpath, sysProps, mainClass) = prepareSubmitEnvironment(appArgs) val childArgsMap = childArgs.grouped(2).map(a => a(0) -> a(1)).toMap - assert(childArgsMap.contains("--primary-java-resource")) - assert(childArgsMap.get("--main-class") === Some("org.SomeClass")) - assert(childArgsMap.get("--arg") === Some("arg1")) + childArgsMap.contains("--primary-java-resource") should be (true) + childArgsMap.get("--main-class") should be (Some("org.SomeClass")) + childArgsMap.get("--arg") should be (Some("arg1")) mainClass should be ("org.apache.spark.deploy.k8s.submit.Client") classpath should have length (0) From 77b851fed3a035ab72bbd29224edc8811cf4afab Mon Sep 17 00:00:00 2001 From: foxish Date: Mon, 6 Nov 2017 11:23:43 -0800 Subject: [PATCH 3/4] Add missing options --- .../test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index e53d608ff192c..e2fa347a19f92 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -389,6 +389,7 @@ class SparkSubmitSuite "--class", "org.SomeClass", "--kubernetes-namespace", "foo", "--driver-memory", "4g", + "--conf", "spark.kubernetes.driver.docker.image=bar", "thejar.jar", "arg1") val appArgs = new SparkSubmitArguments(clArgs) @@ -401,7 +402,9 @@ class SparkSubmitSuite mainClass should be ("org.apache.spark.deploy.k8s.submit.Client") classpath should have length (0) sysProps("spark.executor.memory") should be ("5g") + sysProps("spark.driver.memory") should be ("4g") sysProps("spark.kubernetes.namespace") should be ("foo") + sysProps("spark.kubernetes.driver.docker.image") should be ("bar") } test("handles confs with flag equivalents") { From 0add46a0240d74fb4f746e100ee9a7d6d29d87bd Mon Sep 17 00:00:00 2001 From: foxish Date: Mon, 6 Nov 2017 11:42:59 -0800 Subject: [PATCH 4/4] Added check for jar --- .../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index e2fa347a19f92..faaa60269686d 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -390,15 +390,15 @@ class SparkSubmitSuite "--kubernetes-namespace", "foo", "--driver-memory", "4g", "--conf", "spark.kubernetes.driver.docker.image=bar", - "thejar.jar", + "/home/thejar.jar", "arg1") val appArgs = new SparkSubmitArguments(clArgs) val (childArgs, classpath, sysProps, mainClass) = prepareSubmitEnvironment(appArgs) + val childArgsMap = childArgs.grouped(2).map(a => a(0) -> a(1)).toMap - childArgsMap.contains("--primary-java-resource") should be (true) + childArgsMap.get("--primary-java-resource") should be (Some("file:/home/thejar.jar")) childArgsMap.get("--main-class") should be (Some("org.SomeClass")) childArgsMap.get("--arg") should be (Some("arg1")) - mainClass should be ("org.apache.spark.deploy.k8s.submit.Client") classpath should have length (0) sysProps("spark.executor.memory") should be ("5g")