From 8a265b7f1084e8d211833dc31633f1f2a16195c6 Mon Sep 17 00:00:00 2001 From: witgo Date: Fri, 25 Apr 2014 01:12:10 +0800 Subject: [PATCH 1/6] Fix SPARK-1609: Executor fails to start when use spark-submit --- .../scala/org/apache/spark/deploy/worker/CommandUtils.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala index 9103c885fa96c..e1adced67bc08 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala @@ -62,10 +62,10 @@ object CommandUtils extends Logging { val classPath = Utils.executeAndGetOutput( Seq(sparkHome + "/bin/compute-classpath" + ext), extraEnvironment=command.environment) - val userClassPath = command.classPathEntries.mkString(File.pathSeparator) - val classPathWithUser = classPath + File.pathSeparator + userClassPath + val userClassPath = command.classPathEntries ++ Seq(classPath) - Seq("-cp", classPathWithUser) ++ libraryOpts ++ extraOpts ++ memoryOpts + Seq("-cp", userClassPath.filterNot(_.isEmpty).mkString(File.pathSeparator)) ++ + libraryOpts ++ extraOpts ++ memoryOpts } /** Spawn a thread that will redirect a given stream to a file */ From 86fc4bbae56f937e88595a10a01b3db7770e460b Mon Sep 17 00:00:00 2001 From: witgo Date: Fri, 25 Apr 2014 10:51:54 +0800 Subject: [PATCH 2/6] bugfix --- .../org/apache/spark/deploy/worker/CommandUtils.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala index e1adced67bc08..60e974535e4bd 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala @@ -48,7 +48,13 @@ object CommandUtils extends Logging { def buildJavaOpts(command: Command, memory: Int, sparkHome: String): Seq[String] = { val memoryOpts = Seq(s"-Xms${memory}M", s"-Xmx${memory}M") // Note, this will coalesce multiple options into a single command component - val extraOpts = command.extraJavaOptions.toSeq + val extraOpts = command.extraJavaOptions match { + case Some(opts) => + opts.split(" ") + case _ => + Seq() + } + val libraryOpts = if (command.libraryPathEntries.size > 0) { val joined = command.libraryPathEntries.mkString(File.pathSeparator) From f7c0ab71ceef4023ec2f63f65d0a7d346e989fa0 Mon Sep 17 00:00:00 2001 From: witgo Date: Fri, 25 Apr 2014 10:55:01 +0800 Subject: [PATCH 3/6] bugfix --- .../scala/org/apache/spark/deploy/worker/CommandUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala index 60e974535e4bd..d8b8aa9a330c4 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala @@ -50,7 +50,7 @@ object CommandUtils extends Logging { // Note, this will coalesce multiple options into a single command component val extraOpts = command.extraJavaOptions match { case Some(opts) => - opts.split(" ") + opts.split(" ").toSeq case _ => Seq() } From 1185605f34457767909259f83a8e44be7456d7fe Mon Sep 17 00:00:00 2001 From: witgo Date: Fri, 25 Apr 2014 11:00:29 +0800 Subject: [PATCH 4/6] fix extraJavaOptions split --- .../scala/org/apache/spark/deploy/worker/CommandUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala index d8b8aa9a330c4..fe1cd64511ccc 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala @@ -50,7 +50,7 @@ object CommandUtils extends Logging { // Note, this will coalesce multiple options into a single command component val extraOpts = command.extraJavaOptions match { case Some(opts) => - opts.split(" ").toSeq + opts.split("\\s+").toSeq case _ => Seq() } From 06408523e1166b60646fbb661a3bcbeb22c23260 Mon Sep 17 00:00:00 2001 From: witgo Date: Sat, 26 Apr 2014 02:12:46 +0800 Subject: [PATCH 5/6] review commit --- .../scala/org/apache/spark/deploy/worker/CommandUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala index fe1cd64511ccc..b887f68d26a4c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala @@ -50,7 +50,7 @@ object CommandUtils extends Logging { // Note, this will coalesce multiple options into a single command component val extraOpts = command.extraJavaOptions match { case Some(opts) => - opts.split("\\s+").toSeq + Utils.splitCommandString(opts) case _ => Seq() } From deb6a4c5a412a513d8fc685a299b5e5bfce350bd Mon Sep 17 00:00:00 2001 From: witgo Date: Sat, 26 Apr 2014 13:59:53 +0800 Subject: [PATCH 6/6] review commit --- .../org/apache/spark/deploy/worker/CommandUtils.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala index b887f68d26a4c..3e615e753b342 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala @@ -48,12 +48,7 @@ object CommandUtils extends Logging { def buildJavaOpts(command: Command, memory: Int, sparkHome: String): Seq[String] = { val memoryOpts = Seq(s"-Xms${memory}M", s"-Xmx${memory}M") // Note, this will coalesce multiple options into a single command component - val extraOpts = command.extraJavaOptions match { - case Some(opts) => - Utils.splitCommandString(opts) - case _ => - Seq() - } + val extraOpts = command.extraJavaOptions.map(Utils.splitCommandString).getOrElse(Seq()) val libraryOpts = if (command.libraryPathEntries.size > 0) {