From c76763b875beedba0a144efe1d3b814cfc8b811b Mon Sep 17 00:00:00 2001 From: witgo Date: Sat, 3 May 2014 21:57:09 +0800 Subject: [PATCH 1/6] The default value of yarn.version is equal to hadoop.version --- bin/compute-classpath.sh | 1 + docs/building-with-maven.md | 9 +- pom.xml | 212 +++++++++++++++++++++++++----------- project/SparkBuild.scala | 2 +- yarn/pom.xml | 5 +- 5 files changed, 156 insertions(+), 73 deletions(-) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index b0218531e9eb..70ac71e45db4 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -44,6 +44,7 @@ if [ -f "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar ]; then CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/classes" + CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SCALA_VERSION/classes" DEPS_ASSEMBLY_JAR=`ls "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar` CLASSPATH="$CLASSPATH:$DEPS_ASSEMBLY_JAR" diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index a5e530346740..e447dfea3bac 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -45,17 +45,20 @@ For Apache Hadoop versions 1.x, Cloudera CDH MRv1, and other Hadoop versions wit For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with YARN, you can enable the "yarn-alpha" or "yarn" profile and set the "hadoop.version", "yarn.version" property. Note that Hadoop 0.23.X requires a special `-Phadoop-0.23` profile: # Apache Hadoop 2.0.5-alpha - $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -Dyarn.version=2.0.5-alpha -DskipTests clean package + $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -DskipTests clean package # Cloudera CDH 4.2.0 with MapReduce v2 - $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -Dyarn.version=2.0.0-cdh4.2.0 -DskipTests clean package + $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -DskipTests clean package # Apache Hadoop 2.2.X (e.g. 2.2.0 as below) and newer - $ mvn -Pyarn -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -DskipTests clean package + $ mvn -Pyarn -Dhadoop.version=2.2.0 -DskipTests clean package # Apache Hadoop 0.23.x $ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -Dyarn.version=0.23.7 -DskipTests clean package + # Different versions of HDFS and YARN. + $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version=0.23.7 -DskipTests clean package + ## Spark Tests in Maven ## Tests are run by default via the [ScalaTest Maven plugin](http://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin). Some of the require Spark to be packaged first, so always run `mvn package` with `-DskipTests` the first time. You can then run the tests with `mvn -Dhadoop.version=... test`. diff --git a/pom.xml b/pom.xml index 646753fe3030..b211468b84a8 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,8 @@ ~ limitations under the License. --> - + 4.0.0 org.apache @@ -119,7 +120,7 @@ 1.2.17 1.0.4 2.4.1 - 0.23.7 + ${hadoop.version} 0.94.6 0.12.0 1.3.2 @@ -135,7 +136,8 @@ - maven-repo + maven-repo + Maven Repository http://repo.maven.apache.org/maven2 @@ -558,64 +560,7 @@ jets3t 0.7.1 - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - org.codehaus.jackson @@ -850,12 +795,78 @@ yarn - - - org.apache.avro - avro - - + + + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + javax.servlet + servlet-api + + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + + + @@ -901,7 +912,76 @@ yarn + + + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + + + diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 51f733511116..03e25acf6f1b 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -55,7 +55,7 @@ object SparkBuild extends Build { val SCALAC_JVM_VERSION = "jvm-1.6" val JAVAC_JVM_VERSION = "1.6" - lazy val root = Project("root", file("."), settings = rootSettings) aggregate(allProjects: _*) + lazy val root = Project("spark", file("."), settings = rootSettings) aggregate(allProjects: _*) lazy val core = Project("core", file("core"), settings = coreSettings) diff --git a/yarn/pom.xml b/yarn/pom.xml index 02f36627431b..e0094738266b 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -28,7 +28,7 @@ yarn-parent_2.10 pom Spark Project YARN Parent POM - + org.apache.spark @@ -50,7 +50,6 @@ org.apache.hadoop hadoop-client - ${yarn.version} org.scalatest @@ -128,7 +127,7 @@ - + From 20e7e3ec8c4c078158cdc22372edc15f199ac4ef Mon Sep 17 00:00:00 2001 From: witgo Date: Sun, 4 May 2014 08:23:16 +0800 Subject: [PATCH 2/6] review commit --- pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pom.xml b/pom.xml index b211468b84a8..b4982fd43552 100644 --- a/pom.xml +++ b/pom.xml @@ -950,10 +950,6 @@ org.jboss.netty netty - - javax.servlet - servlet-api - @@ -974,10 +970,6 @@ org.jboss.netty netty - - javax.servlet - servlet-api - From a1d876ae1b7ef77984c159ab53bacf6230b0564b Mon Sep 17 00:00:00 2001 From: witgo Date: Sun, 4 May 2014 08:27:58 +0800 Subject: [PATCH 3/6] review commit --- pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pom.xml b/pom.xml index b4982fd43552..3b7f6db796e2 100644 --- a/pom.xml +++ b/pom.xml @@ -833,12 +833,6 @@ org.jboss.netty netty - - - javax.servlet - servlet-api - @@ -859,10 +853,6 @@ org.jboss.netty netty - - javax.servlet - servlet-api - From 2df6cf5a307723393124149a91ada08d1b09f138 Mon Sep 17 00:00:00 2001 From: witgo Date: Sun, 4 May 2014 08:57:45 +0800 Subject: [PATCH 4/6] review commit --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 03e25acf6f1b..51f733511116 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -55,7 +55,7 @@ object SparkBuild extends Build { val SCALAC_JVM_VERSION = "jvm-1.6" val JAVAC_JVM_VERSION = "1.6" - lazy val root = Project("spark", file("."), settings = rootSettings) aggregate(allProjects: _*) + lazy val root = Project("root", file("."), settings = rootSettings) aggregate(allProjects: _*) lazy val core = Project("core", file("core"), settings = coreSettings) From f8a4ad807abcc9396c35970317f5098d15ab9736 Mon Sep 17 00:00:00 2001 From: witgo Date: Sun, 4 May 2014 10:20:58 +0800 Subject: [PATCH 5/6] revert remove the dependency of avro in yarn-alpha --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 3b7f6db796e2..0625ca8907c3 100644 --- a/pom.xml +++ b/pom.xml @@ -792,6 +792,12 @@ 0.23.7 + + + org.apache.avro + avro + + yarn From c390631b0ce4f6877e02cff74299f7e1bb0da2b3 Mon Sep 17 00:00:00 2001 From: witgo Date: Sun, 4 May 2014 13:33:15 +0800 Subject: [PATCH 6/6] restore the yarn dependency declarations --- pom.xml | 179 ++++++++++++++++++-------------------------------------- 1 file changed, 57 insertions(+), 122 deletions(-) diff --git a/pom.xml b/pom.xml index 0625ca8907c3..ebd359a9de17 100644 --- a/pom.xml +++ b/pom.xml @@ -560,7 +560,64 @@ jets3t 0.7.1 + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + org.codehaus.jackson @@ -801,68 +858,7 @@ yarn - - - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - @@ -908,68 +904,7 @@ yarn - - - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - -