From 9a95a564593ad4071486abb51750cbf6c9b921ff Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 5 May 2014 18:25:04 +0800 Subject: [PATCH 1/4] Add missing description to spark-env.sh.template --- conf/spark-env.sh.template | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index f906be611a931..d20777d0a264d 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -38,6 +38,7 @@ # - SPARK_WORKER_INSTANCES, to set the number of worker processes per node # - SPARK_WORKER_DIR, to set the working directory of worker processes # - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y") +# - SPARK_DRIVER_MEMORY, Memory for Master (e.g. 1000M, 2G) (Default: 512 Mb) # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") # - SPARK_DAEMON_OPTS, to set config properties for all daemons (e.g. "-Dx=y") # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers From 6a572cabd71c26e0553e1adc04b0e53839a623ee Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 8 May 2014 10:15:29 +0800 Subject: [PATCH 2/4] merge master --- conf/spark-env.sh.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index d20777d0a264d..330300c010ae5 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -38,7 +38,7 @@ # - SPARK_WORKER_INSTANCES, to set the number of worker processes per node # - SPARK_WORKER_DIR, to set the working directory of worker processes # - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y") -# - SPARK_DRIVER_MEMORY, Memory for Master (e.g. 1000M, 2G) (Default: 512 Mb) +# - SPARK_DRIVER_MEMORY, Memory for driver (e.g. 1000M, 2G) (Default: 512 Mb) # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") # - SPARK_DAEMON_OPTS, to set config properties for all daemons (e.g. "-Dx=y") # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers From 1538ae354dfa991d5081f474ef9f3f3ef82c29f0 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 8 May 2014 12:03:48 +0800 Subject: [PATCH 3/4] review commit --- project/SparkBuild.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 7f9746ec4acc0..2a62d8d9e87fb 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -573,7 +573,14 @@ object SparkBuild extends Build { name := "spark-assembly", assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }, - jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } + jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } , + mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) => + { + case PathList("org", "datanucleus", xs @ _*) => MergeStrategy.discard + case x => old(x) + } + } + ) ++ assemblySettings ++ extraAssemblySettings def extraAssemblySettings() = Seq( From f3d2d5b029cb5c76292a5c8e27856d545d90d3fe Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 8 May 2014 12:55:04 +0800 Subject: [PATCH 4/4] Reducing errors submit --- project/SparkBuild.scala | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 2a62d8d9e87fb..7f9746ec4acc0 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -573,14 +573,7 @@ object SparkBuild extends Build { name := "spark-assembly", assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }, - jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } , - mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) => - { - case PathList("org", "datanucleus", xs @ _*) => MergeStrategy.discard - case x => old(x) - } - } - + jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings def extraAssemblySettings() = Seq(