From 5119627e4b7ce5547d188bbaad530858223d637c Mon Sep 17 00:00:00 2001 From: Iulian Dragos Date: Tue, 19 Jan 2016 16:34:15 +0100 Subject: [PATCH] =?UTF-8?q?Don=E2=80=99t=20pass=20deployMode=20and=20spark?= =?UTF-8?q?.master=20when=20launching=20a=20driver.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the launched driver should be in client mode, but at submission time this setting is set to ‘cluster’. - the `spark.master` setting is set to the dispatcher address (usually on port 7077). Passing it has no effect, since we also pass `—-master` which takes precedence. But this is confusing so we should filter it out. --- .../spark/scheduler/cluster/mesos/MesosClusterScheduler.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala index 940025a7c06d..82e69d0aa653 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala @@ -424,7 +424,9 @@ private[spark] class MesosClusterScheduler( "--driver-memory", s"${desc.mem}M") val replicatedOptionsBlacklist = Set( - "spark.jars" // Avoids duplicate classes in classpath + "spark.jars", // Avoids duplicate classes in classpath + "spark.submit.deployMode", // this would be set to `cluster`, but we need client + "spark.master" // this contains the address of the dispatcher, not master ) // Assume empty main class means we're running python