From e4c5fb12d732029727ba6772732734c511c1f6fa Mon Sep 17 00:00:00 2001 From: Marcin Kurczych Date: Tue, 20 Feb 2018 16:12:09 +0100 Subject: [PATCH 1/2] [SPARK-23464][MESOS] Fix mesos cluster scheduler options double-escaping Don't enclose --conf option value with "", as they are already escaped by shellEscape and additional wrapping cancels out this escaping, causing driver fail to start. This reverts commit 9b377aa49f14af31f54164378d60e0fdea2142e5 [SPARK-18114]. Manual test with driver command logging added. Author: Marcin Kurczych --- .../spark/scheduler/cluster/mesos/MesosClusterScheduler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala index d224a7325820..3b4e3d5d9cb5 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala @@ -530,7 +530,7 @@ private[spark] class MesosClusterScheduler( .filter { case (key, _) => !replicatedOptionsBlacklist.contains(key) } .toMap (defaultConf ++ driverConf).foreach { case (key, value) => - options ++= Seq("--conf", s""""$key=${shellEscape(value)}"""".stripMargin) } + options ++= Seq("--conf", s"$key=${shellEscape(value)}") } options } From 22c67392b9f920af8fae9de1f7c7f5fb5a25abb8 Mon Sep 17 00:00:00 2001 From: Marcin Kurczych Date: Fri, 2 Mar 2018 00:09:33 +0100 Subject: [PATCH 2/2] [SPARK-23464][MESOS] Add unit test [to be squashed] --- .../mesos/MesosClusterSchedulerSuite.scala | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala index e534b9d7e3ed..9d550a030f1b 100644 --- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala +++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala @@ -199,6 +199,38 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi }) } + test("properly wraps and escapes parameters passed to driver command") { + setScheduler() + + val mem = 1000 + val cpu = 1 + + val response = scheduler.submitDriver( + new MesosDriverDescription("d1", "jar", mem, cpu, true, + command, + Map("spark.mesos.executor.home" -> "test", + "spark.app.name" -> "test", + // no special characters, wrap only + "spark.driver.extraJavaOptions" -> + "-XX+PrintGC -Dparam1=val1 -Dparam2=val2", + // special characters, to be escaped + "spark.executor.extraJavaOptions" -> + """-Dparam1="value 1" -Dparam2=value\ 2 -Dpath=$PATH"""), + "s1", + new Date())) + assert(response.success) + + val offer = Utils.createOffer("o1", "s1", mem, cpu) + scheduler.resourceOffers(driver, List(offer).asJava) + val tasks = Utils.verifyTaskLaunched(driver, "o1") + val driverCmd = tasks.head.getCommand.getValue + assert(driverCmd.contains( + """--conf spark.driver.extraJavaOptions="-XX+PrintGC -Dparam1=val1 -Dparam2=val2"""")) + assert(driverCmd.contains( + """--conf spark.executor.extraJavaOptions=""" + + """"-Dparam1=\"value 1\" -Dparam2=value\\ 2 -Dpath=\$PATH"""")) + } + test("supports spark.mesos.driverEnv.*") { setScheduler()