From a24aff91d59dd848cdd5802602b340db832e5507 Mon Sep 17 00:00:00 2001 From: "U-CCR\\daianyue" Date: Fri, 21 Oct 2016 11:15:46 +0800 Subject: [PATCH 1/3] Added support for extra command in front of spark. Such as numactl etc. --- conf/spark-env.sh.template | 1 + .../org/apache/spark/deploy/yarn/ExecutorRunnable.scala | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index 5c1e876ef9afc..b355adf177d9e 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -40,6 +40,7 @@ # - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1). # - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G) # - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G) +# - SPARK_COMMAND_PREFIX, Command string added in front of normal spark comamnds # Options for the daemons used in the standalone deploy mode # - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 8e0533f39ae53..1cdb98740de36 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -207,8 +207,12 @@ private[yarn] class ExecutorRunnable( }.toSeq YarnSparkHadoopUtil.addOutOfMemoryErrorArgument(javaOpts) + + // Add support for extra executor prefix. + val commandPrefix = (if (sys.env.contains("SPARK_COMMAND_PREFIX")) sys.env.get("SPARK_COMMAND_PREFIX") else "") + val commands = prefixEnv ++ Seq( - YarnSparkHadoopUtil.expandEnvironment(Environment.JAVA_HOME) + "/bin/java", + commandPrefix + " " + YarnSparkHadoopUtil.expandEnvironment(Environment.JAVA_HOME) + "/bin/java", "-server") ++ javaOpts ++ Seq("org.apache.spark.executor.CoarseGrainedExecutorBackend", From b6780ccf8e8739c08f4e7b7c73a20d32119aa0d0 Mon Sep 17 00:00:00 2001 From: sheepduke Date: Thu, 27 Oct 2016 13:34:53 +0800 Subject: [PATCH 2/3] Modifed source code to meet Spark code style. --- .../org/apache/spark/deploy/yarn/ExecutorRunnable.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 1cdb98740de36..97cf2b08e8f99 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -209,10 +209,12 @@ private[yarn] class ExecutorRunnable( YarnSparkHadoopUtil.addOutOfMemoryErrorArgument(javaOpts) // Add support for extra executor prefix. - val commandPrefix = (if (sys.env.contains("SPARK_COMMAND_PREFIX")) sys.env.get("SPARK_COMMAND_PREFIX") else "") + val commandPrefix = (if (sys.env.contains("SPARK_COMMAND_PREFIX")) + sys.env.get("SPARK_COMMAND_PREFIX") else ""); val commands = prefixEnv ++ Seq( - commandPrefix + " " + YarnSparkHadoopUtil.expandEnvironment(Environment.JAVA_HOME) + "/bin/java", + commandPrefix + " " + YarnSparkHadoopUtil.expandEnvironment(Environment.JAVA_HOME) + + "/bin/java", "-server") ++ javaOpts ++ Seq("org.apache.spark.executor.CoarseGrainedExecutorBackend", From d3628ebb3921490c186fa339881c4213f916809b Mon Sep 17 00:00:00 2001 From: sheepduke Date: Mon, 7 Nov 2016 11:40:25 +0800 Subject: [PATCH 3/3] Changed environment varaible. --- conf/spark-env.sh.template | 2 +- .../org/apache/spark/deploy/yarn/ExecutorRunnable.scala | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index b355adf177d9e..7b9c03acd2adc 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -40,7 +40,7 @@ # - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1). # - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G) # - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G) -# - SPARK_COMMAND_PREFIX, Command string added in front of normal spark comamnds +# - SPARK_EXECUTOR_LAUNCH_PREFIX, Command string added in front of Spark executor # Options for the daemons used in the standalone deploy mode # - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 97cf2b08e8f99..8df870b3fd852 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -193,7 +193,8 @@ private[yarn] class ExecutorRunnable( */ // For log4j configuration to reference - javaOpts += ("-Dspark.yarn.app.container.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR) + javaOpts += ("-Dspark.yarn.app.container.log.dir=" + + ApplicationConstants.LOG_DIR_EXPANSION_VAR) YarnCommandBuilderUtils.addPermGenSizeOpt(javaOpts) val userClassPath = Client.getUserClasspath(sparkConf).flatMap { uri => @@ -209,11 +210,11 @@ private[yarn] class ExecutorRunnable( YarnSparkHadoopUtil.addOutOfMemoryErrorArgument(javaOpts) // Add support for extra executor prefix. - val commandPrefix = (if (sys.env.contains("SPARK_COMMAND_PREFIX")) - sys.env.get("SPARK_COMMAND_PREFIX") else ""); + val executorCommandPrefix = (if (sys.env.contains("SPARK_EXECUTOR_LAUNCH_PREFIX")) + sys.env.get("SPARK_EXECUTOR_LAUNCH_PREFIX") else ""); val commands = prefixEnv ++ Seq( - commandPrefix + " " + YarnSparkHadoopUtil.expandEnvironment(Environment.JAVA_HOME) + + executorCommandPrefix + " " + YarnSparkHadoopUtil.expandEnvironment(Environment.JAVA_HOME) + "/bin/java", "-server") ++ javaOpts ++