From 6851becba7058571816848326713fa8d08998e5d Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 17 Apr 2014 21:45:48 +0800 Subject: [PATCH 01/10] Maintain consistent SparkBuild.scala, pom.xml --- project/SparkBuild.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 09b527c76a5ae..334d5e1a4ae26 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -216,7 +216,7 @@ object SparkBuild extends Build { org.apache apache - 13 + 14 http://spark.apache.org/ @@ -242,7 +242,7 @@ object SparkBuild extends Build { JIRA - https://spark-project.atlassian.net/browse/SPARK + https://issues.apache.org/jira/browse/SPARK ), From 0c6c1fc4c6005def6391beadd55e463cc5b65344 Mon Sep 17 00:00:00 2001 From: witgo Date: Tue, 22 Apr 2014 15:29:53 +0800 Subject: [PATCH 02/10] Fix compile spark core error with hadoop 0.23.x --- bagel/pom.xml | 14 --- core/pom.xml | 13 --- examples/pom.xml | 18 +--- external/flume/pom.xml | 14 --- external/kafka/pom.xml | 14 --- external/mqtt/pom.xml | 14 --- external/twitter/pom.xml | 14 --- external/zeromq/pom.xml | 14 --- graphx/pom.xml | 14 --- mllib/pom.xml | 14 --- pom.xml | 203 +++++++++++++++++++++++++++++++++++++-- project/SparkBuild.scala | 2 +- repl/pom.xml | 14 --- sql/catalyst/pom.xml | 109 ++++++++++----------- sql/core/pom.xml | 122 +++++++++++------------ sql/hive/pom.xml | 186 +++++++++++++++++------------------ streaming/pom.xml | 14 --- tools/pom.xml | 14 --- yarn/alpha/pom.xml | 14 --- yarn/pom.xml | 9 -- yarn/stable/pom.xml | 14 --- 21 files changed, 394 insertions(+), 450 deletions(-) diff --git a/bagel/pom.xml b/bagel/pom.xml index 142f75c5d2c64..355f437c5b16a 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -31,20 +31,6 @@ Spark Project Bagel http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/core/pom.xml b/core/pom.xml index d87e2bca030e3..735128180dc14 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -30,19 +30,6 @@ jar Spark Project Core http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.hadoop diff --git a/examples/pom.xml b/examples/pom.xml index 0b6212b5d1549..cde35ae8dc5b1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -31,20 +31,6 @@ Spark Project Examples http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark @@ -124,6 +110,10 @@ commons-logging commons-logging + + org.jruby + jruby-complete + diff --git a/external/flume/pom.xml b/external/flume/pom.xml index f21963531574b..bc00ab4573c39 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -31,20 +31,6 @@ Spark Project External Flume http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index 343e1fabd823f..979eb0ca624bd 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -31,20 +31,6 @@ Spark Project External Kafka http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml index 3710a63541d78..9aa1c1a9f5b80 100644 --- a/external/mqtt/pom.xml +++ b/external/mqtt/pom.xml @@ -31,20 +31,6 @@ Spark Project External MQTT http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml index 398b9f4fbaa7d..a443459594710 100644 --- a/external/twitter/pom.xml +++ b/external/twitter/pom.xml @@ -31,20 +31,6 @@ Spark Project External Twitter http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml index 77e957f404645..a40e55876e640 100644 --- a/external/zeromq/pom.xml +++ b/external/zeromq/pom.xml @@ -31,20 +31,6 @@ Spark Project External ZeroMQ http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/graphx/pom.xml b/graphx/pom.xml index b4c67ddcd8ca9..dc108d2fe7fbd 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -31,20 +31,6 @@ Spark Project GraphX http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/mllib/pom.xml b/mllib/pom.xml index e7ce00efc4af6..cdd33dbb7970d 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -31,20 +31,6 @@ Spark Project ML Library http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/pom.xml b/pom.xml index 4ff18afa227be..627f70b8ec8b7 100644 --- a/pom.xml +++ b/pom.xml @@ -293,7 +293,7 @@ org.ow2.asm asm-commons - + @@ -308,7 +308,7 @@ org.ow2.asm asm-commons - + @@ -518,6 +518,44 @@ + + org.apache.avro + avro-ipc + 1.7.4 + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + org.mortbay.jetty + servlet-api + + + org.apache.velocity + velocity + + + + + org.apache.avro + avro-mapred + 1.7.4 + + + org.jboss.netty + netty + + + io.netty + netty + + + net.java.dev.jets3t jets3t @@ -701,7 +739,7 @@ . ${project.build.directory}/SparkTestSuite.txt -Xmx3g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=512m - + @@ -756,7 +794,7 @@ build-helper-maven-plugin - add-scala-sources + add-scala-sources generate-sources add-source @@ -793,6 +831,157 @@ + + + hadoop0.23.1 + + + hadoop.version + 0.23.1 + + + + + org.apache.avro + avro + + + + + hadoop0.23.2 + + + hadoop.version + 0.23.2 + + + + + org.apache.avro + avro + + + + + hadoop0.23.3 + + + hadoop.version + 0.23.3 + + + + + org.apache.avro + avro + + + + + hadoop0.23.4 + + + hadoop.version + 0.23.4 + + + + + org.apache.avro + avro + + + + + hadoop0.23.5 + + + hadoop.version + 0.23.5 + + + + + org.apache.avro + avro + + + + + hadoop0.23.6 + + + hadoop.version + 0.23.6 + + + + + org.apache.avro + avro + + + + + hadoop0.23.7 + + + hadoop.version + 0.23.7 + + + + + org.apache.avro + avro + + + + + hadoop0.23.8 + + + hadoop.version + 0.23.8 + + + + + org.apache.avro + avro + + + + + hadoop0.23.9 + + + hadoop.version + 0.23.9 + + + + + org.apache.avro + avro + + + + + hadoop0.23.10 + + + hadoop.version + 0.23.10 + + + + + org.apache.avro + avro + + + yarn-alpha @@ -851,8 +1040,8 @@ - - + + hadoop-provided @@ -896,6 +1085,6 @@ - + diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f115f0d3a041c..b051d0cdbae4d 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -106,7 +106,7 @@ object SparkBuild extends Build { case Some(v) => v.toBoolean } lazy val hadoopClient = if (hadoopVersion.startsWith("0.20.") || hadoopVersion == "1.0.0") "hadoop-core" else "hadoop-client" - val maybeAvro = if (hadoopVersion.startsWith("0.23.") && isYarnEnabled) Seq("org.apache.avro" % "avro" % "1.7.4") else Seq() + val maybeAvro = if (hadoopVersion.startsWith("0.23.")) Seq("org.apache.avro" % "avro" % "1.7.4") else Seq() lazy val isHiveEnabled = Properties.envOrNone("SPARK_HIVE") match { case None => DEFAULT_HIVE diff --git a/repl/pom.xml b/repl/pom.xml index 78d2fe13c27eb..b761a176ce256 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -31,20 +31,6 @@ Spark Project REPL http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - /usr/share/spark root diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 9d5c6a857bb00..8d2e4baf69e30 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -16,67 +16,56 @@ ~ limitations under the License. --> - - 4.0.0 - - org.apache.spark - spark-parent - 1.0.0-SNAPSHOT - ../../pom.xml - - + + 4.0.0 + org.apache.spark - spark-catalyst_2.10 - jar - Spark Project Catalyst - http://spark.apache.org/ + spark-parent + 1.0.0-SNAPSHOT + ../../pom.xml + - - - yarn-alpha - - - org.apache.avro - avro - - - - + org.apache.spark + spark-catalyst_2.10 + jar + Spark Project Catalyst + http://spark.apache.org/ - - - org.scala-lang - scala-reflect - - - org.apache.spark - spark-core_${scala.binary.version} - ${project.version} - - - com.typesafe - scalalogging-slf4j_${scala.binary.version} - 1.0.1 - - - org.scalatest - scalatest_${scala.binary.version} - test - - - org.scalacheck - scalacheck_${scala.binary.version} - test - - - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - - + + + org.scala-lang + scala-reflect + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + + + com.typesafe + scalalogging-slf4j_${scala.binary.version} + 1.0.1 + + + org.scalatest + scalatest_${scala.binary.version} + test + + + org.scalacheck + scalacheck_${scala.binary.version} + test + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + + org.scalatest + scalatest-maven-plugin + + + diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 85580ed6b822f..fb3b190b4ec5a 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -16,72 +16,62 @@ ~ limitations under the License. --> - - 4.0.0 - - org.apache.spark - spark-parent - 1.0.0-SNAPSHOT - ../../pom.xml - - + + 4.0.0 + org.apache.spark - spark-sql_2.10 - jar - Spark Project SQL - http://spark.apache.org/ - - - yarn-alpha - - - org.apache.avro - avro - - - - + spark-parent + 1.0.0-SNAPSHOT + ../../pom.xml + + + org.apache.spark + spark-sql_2.10 + jar + Spark Project SQL + http://spark.apache.org/ - - - org.apache.spark - spark-core_${scala.binary.version} - ${project.version} - - - org.apache.spark - spark-catalyst_${scala.binary.version} - ${project.version} - - - com.twitter - parquet-column - ${parquet.version} - - - com.twitter - parquet-hadoop - ${parquet.version} - - - org.scalatest - scalatest_${scala.binary.version} - test - - - org.scalacheck - scalacheck_${scala.binary.version} - test - - - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - - + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${project.version} + + + com.twitter + parquet-column + ${parquet.version} + + + com.twitter + parquet-hadoop + ${parquet.version} + + + org.scalatest + scalatest_${scala.binary.version} + test + + + org.scalacheck + scalacheck_${scala.binary.version} + test + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + + org.scalatest + scalatest-maven-plugin + + + diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index a662da76ce25a..889d249146b8c 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -16,105 +16,95 @@ ~ limitations under the License. --> - - 4.0.0 - - org.apache.spark - spark-parent - 1.0.0-SNAPSHOT - ../../pom.xml - - + + 4.0.0 + org.apache.spark - spark-hive_2.10 - jar - Spark Project Hive - http://spark.apache.org/ - - - yarn-alpha - - - org.apache.avro - avro - - - - + spark-parent + 1.0.0-SNAPSHOT + ../../pom.xml + + + org.apache.spark + spark-hive_2.10 + jar + Spark Project Hive + http://spark.apache.org/ - - - org.apache.spark - spark-core_${scala.binary.version} - ${project.version} - - - org.apache.spark - spark-sql_${scala.binary.version} - ${project.version} - - - org.apache.hive - hive-metastore - ${hive.version} - - - org.apache.hive - hive-exec - ${hive.version} - - - org.codehaus.jackson - jackson-mapper-asl - - - org.apache.hive - hive-serde - ${hive.version} - - - org.scalatest - scalatest_${scala.binary.version} - test - - - org.scalacheck - scalacheck_${scala.binary.version} - test - - - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + ${project.version} + + + org.apache.hive + hive-metastore + ${hive.version} + + + org.apache.hive + hive-exec + ${hive.version} + + + org.codehaus.jackson + jackson-mapper-asl + + + org.apache.hive + hive-serde + ${hive.version} + + + org.scalatest + scalatest_${scala.binary.version} + test + + + org.scalacheck + scalacheck_${scala.binary.version} + test + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + + org.scalatest + scalatest-maven-plugin + - - - org.apache.maven.plugins - maven-dependency-plugin - 2.4 - - - copy-dependencies - package - - copy-dependencies - - - - ${basedir}/../../lib_managed/jars - false - false - true - org.datanucleus - - - - - - + + + org.apache.maven.plugins + maven-dependency-plugin + 2.4 + + + copy-dependencies + package + + copy-dependencies + + + + ${basedir}/../../lib_managed/jars + false + false + true + org.datanucleus + + + + + + diff --git a/streaming/pom.xml b/streaming/pom.xml index 93b1c5a37aff9..6435224a14674 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -31,20 +31,6 @@ Spark Project Streaming http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/tools/pom.xml b/tools/pom.xml index ae2ba64e07c21..1875c497bc61c 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -30,20 +30,6 @@ Spark Project Tools http://spark.apache.org/ - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark diff --git a/yarn/alpha/pom.xml b/yarn/alpha/pom.xml index d0aeaceb0d23c..e076ca1d44b97 100644 --- a/yarn/alpha/pom.xml +++ b/yarn/alpha/pom.xml @@ -24,20 +24,6 @@ ../pom.xml - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark spark-yarn-alpha_2.10 jar diff --git a/yarn/pom.xml b/yarn/pom.xml index 3342cb65edcd1..02f36627431b9 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -70,15 +70,6 @@ alpha - - - - - org.apache.avro - avro - - diff --git a/yarn/stable/pom.xml b/yarn/stable/pom.xml index e7915d12aef63..0780f251b595c 100644 --- a/yarn/stable/pom.xml +++ b/yarn/stable/pom.xml @@ -24,20 +24,6 @@ ../pom.xml - - - - yarn-alpha - - - org.apache.avro - avro - - - - - org.apache.spark spark-yarn_2.10 jar From 99464d2787d566153d5c896ade50a670fff9206f Mon Sep 17 00:00:00 2001 From: witgo Date: Tue, 22 Apr 2014 22:29:21 +0800 Subject: [PATCH 03/10] fix maven hadoop-provided profile fails to build --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 627f70b8ec8b7..68b2ab31a01e9 100644 --- a/pom.xml +++ b/pom.xml @@ -1081,6 +1081,7 @@ org.apache.zookeeper zookeeper + 3.4.5 provided From cf46edc3c7d5bd4fe0c917030082c9f9c21880c9 Mon Sep 17 00:00:00 2001 From: witgo Date: Tue, 22 Apr 2014 23:28:52 +0800 Subject: [PATCH 04/10] exclude jruby --- project/SparkBuild.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 3a7188c836027..a41f300c53aca 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -313,6 +313,7 @@ object SparkBuild extends Build { val excludeCurator = ExclusionRule(organization = "org.apache.curator") val excludePowermock = ExclusionRule(organization = "org.powermock") val excludeFastutil = ExclusionRule(organization = "it.unimi.dsi") + val excludeJruby = ExclusionRule(organization = "org.jruby") def sparkPreviousArtifact(id: String, organization: String = "org.apache.spark", version: String = "0.9.0-incubating", crossVersion: String = "2.10"): Option[sbt.ModuleID] = { @@ -414,7 +415,7 @@ object SparkBuild extends Build { name := "spark-examples", libraryDependencies ++= Seq( "com.twitter" %% "algebird-core" % "0.1.11", - "org.apache.hbase" % "hbase" % HBASE_VERSION excludeAll(excludeNetty, excludeAsm, excludeOldAsm, excludeCommonsLogging), + "org.apache.hbase" % "hbase" % HBASE_VERSION excludeAll(excludeNetty, excludeAsm, excludeOldAsm, excludeCommonsLogging, excludeJruby), "org.apache.cassandra" % "cassandra-all" % "1.2.6" exclude("com.google.guava", "guava") exclude("com.googlecode.concurrentlinkedhashmap", "concurrentlinkedhashmap-lru") From 934f24d95b04e644f339f69f4ee38263a00dbe3e Mon Sep 17 00:00:00 2001 From: witgo Date: Tue, 22 Apr 2014 23:46:59 +0800 Subject: [PATCH 05/10] review commit --- docs/building-with-maven.md | 6 ++ pom.xml | 151 ++---------------------------------- 2 files changed, 12 insertions(+), 145 deletions(-) diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index 9cebaf12283fc..7f9070b931502 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -39,6 +39,9 @@ For Apache Hadoop versions 1.x, Cloudera CDH MRv1, and other Hadoop versions wit # Cloudera CDH 4.2.0 with MapReduce v1 $ mvn -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -DskipTests clean package + # Apache Hadoop 0.23.x + $ mvn -Phadoop-0.23 -Dhadoop.version=0.23.7 -DskipTests clean package + For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with YARN, you should enable the "yarn-alpha" or "yarn" profile and set the "hadoop.version", "yarn.version" property: # Apache Hadoop 2.0.5-alpha @@ -50,6 +53,9 @@ For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with # Apache Hadoop 2.2.X ( e.g. 2.2.0 as below ) and newer $ mvn -Pyarn -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -DskipTests clean package + # Apache Hadoop 0.23.x + $ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -Dyarn.version=0.23.7 -DskipTests clean package + ## Spark Tests in Maven ## Tests are run by default via the [ScalaTest Maven plugin](http://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin). Some of the require Spark to be packaged first, so always run `mvn package` with `-DskipTests` the first time. You can then run the tests with `mvn -Dhadoop.version=... test`. diff --git a/pom.xml b/pom.xml index 68b2ab31a01e9..404b1e6d34ab1 100644 --- a/pom.xml +++ b/pom.xml @@ -127,6 +127,7 @@ 8.1.14.v20131031 0.3.1 3.0.0 + 1.7.4 64m 512m @@ -506,7 +507,7 @@ org.apache.avro avro - 1.7.4 + ${avro.version} org.jboss.netty @@ -521,7 +522,7 @@ org.apache.avro avro-ipc - 1.7.4 + ${avro.version} org.mortbay.jetty @@ -544,7 +545,7 @@ org.apache.avro avro-mapred - 1.7.4 + ${avro.version} org.jboss.netty @@ -833,148 +834,7 @@ - hadoop0.23.1 - - - hadoop.version - 0.23.1 - - - - - org.apache.avro - avro - - - - - hadoop0.23.2 - - - hadoop.version - 0.23.2 - - - - - org.apache.avro - avro - - - - - hadoop0.23.3 - - - hadoop.version - 0.23.3 - - - - - org.apache.avro - avro - - - - - hadoop0.23.4 - - - hadoop.version - 0.23.4 - - - - - org.apache.avro - avro - - - - - hadoop0.23.5 - - - hadoop.version - 0.23.5 - - - - - org.apache.avro - avro - - - - - hadoop0.23.6 - - - hadoop.version - 0.23.6 - - - - - org.apache.avro - avro - - - - - hadoop0.23.7 - - - hadoop.version - 0.23.7 - - - - - org.apache.avro - avro - - - - - hadoop0.23.8 - - - hadoop.version - 0.23.8 - - - - - org.apache.avro - avro - - - - - hadoop0.23.9 - - - hadoop.version - 0.23.9 - - - - - org.apache.avro - avro - - - - - hadoop0.23.10 - - - hadoop.version - 0.23.10 - - + hadoop-0.23 org.apache.avro @@ -982,6 +842,7 @@ + yarn-alpha From 62d0862e392331e8db4f5e363fde6ef8ebaefd93 Mon Sep 17 00:00:00 2001 From: witgo Date: Wed, 23 Apr 2014 09:45:10 +0800 Subject: [PATCH 06/10] Fix org.scala-lang: * inconsistent versions dependency --- core/pom.xml | 9 --------- pom.xml | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 735128180dc14..3e9b8a8a652f5 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -134,15 +134,6 @@ org.json4s json4s-jackson_${scala.binary.version} 3.2.6 - - - - org.scala-lang - scalap - - colt diff --git a/pom.xml b/pom.xml index 404b1e6d34ab1..b6f8efb3a275e 100644 --- a/pom.xml +++ b/pom.xml @@ -385,15 +385,6 @@ net.liftweb lift-json_${scala.binary.version} 2.5.1 - - - - org.scala-lang - scalap - - com.codahale.metrics @@ -440,6 +431,16 @@ scala-library ${scala.version} + + org.scala-lang + scala-actors + ${scala.version} + + + org.scala-lang + scalap + ${scala.version} + org.scalatest scalatest_${scala.binary.version} From e34591902770765c755f62fe53237e7250c2c234 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 24 Apr 2014 12:48:50 +0800 Subject: [PATCH 07/10] add avro dependency to yarn-alpha --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index b6f8efb3a275e..988bdbcd7c8f0 100644 --- a/pom.xml +++ b/pom.xml @@ -855,6 +855,12 @@ yarn + + + org.apache.avro + avro + + From 6902c91bdabab0775d4d45a6848047ae138ffeb3 Mon Sep 17 00:00:00 2001 From: witgo Date: Fri, 25 Apr 2014 18:55:16 +0800 Subject: [PATCH 08/10] fix SPARK-1629: Spark Core missing commons-lang dependence --- core/pom.xml | 4 ++++ pom.xml | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/core/pom.xml b/core/pom.xml index 73f573a414050..687e256c03621 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -113,6 +113,10 @@ commons-net commons-net + + commons-lang + commons-lang + ${akka.group} akka-remote_${scala.binary.version} diff --git a/pom.xml b/pom.xml index f1b91133051fe..d4b9629ad3c32 100644 --- a/pom.xml +++ b/pom.xml @@ -371,6 +371,11 @@ commons-net 2.2 + + commons-lang + commons-lang + 2.6 + io.netty netty-all From bee920da40b23cb322085ab36001257ae22b7baa Mon Sep 17 00:00:00 2001 From: witgo Date: Sun, 27 Apr 2014 10:23:27 +0800 Subject: [PATCH 09/10] revert fix SPARK-1629: Spark Core missing commons-lang dependence --- core/pom.xml | 4 ---- pom.xml | 5 ----- 2 files changed, 9 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 687e256c03621..73f573a414050 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -113,10 +113,6 @@ commons-net commons-net - - commons-lang - commons-lang - ${akka.group} akka-remote_${scala.binary.version} diff --git a/pom.xml b/pom.xml index 7890f9259ba99..646753fe30301 100644 --- a/pom.xml +++ b/pom.xml @@ -371,11 +371,6 @@ commons-net 2.2 - - commons-lang - commons-lang - 2.6 - io.netty netty-all From 03f652f35655ee4adf735f73bcc9bfd79e522e25 Mon Sep 17 00:00:00 2001 From: witgo Date: Tue, 29 Apr 2014 13:06:23 +0800 Subject: [PATCH 10/10] review commit --- docs/building-with-maven.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index 748c56726ae9e..e5cd6e0f5600a 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -42,7 +42,7 @@ For Apache Hadoop versions 1.x, Cloudera CDH MRv1, and other Hadoop versions wit # Apache Hadoop 0.23.x $ mvn -Phadoop-0.23 -Dhadoop.version=0.23.7 -DskipTests clean package -For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with YARN, you should enable the "yarn-alpha" or "yarn" profile and set the "hadoop.version", "yarn.version" property: +For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with YARN, you can enable the "yarn-alpha" or "yarn" profile and set the "hadoop.version", "yarn.version" property: # Apache Hadoop 2.0.5-alpha $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -Dyarn.version=2.0.5-alpha -DskipTests clean package