From bbfbbdf5dd3141f98168555abc00412a63544e62 Mon Sep 17 00:00:00 2001 From: Michael Gummelt Date: Thu, 17 Dec 2015 14:37:14 -0800 Subject: [PATCH 1/3] [SPARK-12413] Fix Mesos ZK persistence --- .../org/apache/spark/deploy/rest/mesos/MesosRestServer.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala index c0b93596508f1..b36789569032e 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala @@ -99,7 +99,10 @@ private[mesos] class MesosSubmitRequestServlet( // cause spark-submit script to look for files in SPARK_HOME instead. // We only need the ability to specify where to find spark-submit script // which user can user spark.executor.home or spark.home configurations. - val environmentVariables = request.environmentVariables.filterKeys(!_.equals("SPARK_HOME")) + // + // Due to https://issues.scala-lang.org/browse/SI-6654, `filterKeys` returns an + // unserializable object,so we must call `.map(identity)` on the result. + val environmentVariables = request.environmentVariables.filterKeys(!_.equals("SPARK_HOME")).map(identity) val name = request.sparkProperties.get("spark.app.name").getOrElse(mainClass) // Construct driver description From 05c4da3b41e9efc15703e33544983e41d271685b Mon Sep 17 00:00:00 2001 From: Michael Gummelt Date: Thu, 17 Dec 2015 14:53:45 -0800 Subject: [PATCH 2/3] comment spacing and line length fix --- .../org/apache/spark/deploy/rest/mesos/MesosRestServer.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala index b36789569032e..42ff607aca307 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala @@ -101,8 +101,9 @@ private[mesos] class MesosSubmitRequestServlet( // which user can user spark.executor.home or spark.home configurations. // // Due to https://issues.scala-lang.org/browse/SI-6654, `filterKeys` returns an - // unserializable object,so we must call `.map(identity)` on the result. - val environmentVariables = request.environmentVariables.filterKeys(!_.equals("SPARK_HOME")).map(identity) + // unserializable object, so we must call `.map(identity)` on the result. + val environmentVariables = request.environmentVariables.filterKeys(!_.equals("SPARK_HOME")) + .map(identity) val name = request.sparkProperties.get("spark.app.name").getOrElse(mainClass) // Construct driver description From d70bceebcb204ee4eb486d4d2a02abfb496449ee Mon Sep 17 00:00:00 2001 From: Michael Gummelt Date: Thu, 17 Dec 2015 15:11:50 -0800 Subject: [PATCH 3/3] use filter instead of filterKeys --- .../apache/spark/deploy/rest/mesos/MesosRestServer.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala index 42ff607aca307..87d0fa8b52fe1 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala @@ -100,10 +100,10 @@ private[mesos] class MesosSubmitRequestServlet( // We only need the ability to specify where to find spark-submit script // which user can user spark.executor.home or spark.home configurations. // - // Due to https://issues.scala-lang.org/browse/SI-6654, `filterKeys` returns an - // unserializable object, so we must call `.map(identity)` on the result. - val environmentVariables = request.environmentVariables.filterKeys(!_.equals("SPARK_HOME")) - .map(identity) + // Do not use `filterKeys` here to avoid SI-6654, which breaks ZK persistence + val environmentVariables = request.environmentVariables.filter { case (k, _) => + k != "SPARK_HOME" + } val name = request.sparkProperties.get("spark.app.name").getOrElse(mainClass) // Construct driver description