From 4e96c0153063b35fc03e497f28292a97832e81d4 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 14:03:30 -0700 Subject: [PATCH 01/25] Add YARN/Stable compiled classes to the CLASSPATH. The change adds the `./yarn/stable/target//classes` to the _Classpath_ when a _dependencies_ assembly is available at the assembly directory. Why is this change necessary? Ease the development features and bug-fixes for Spark-YARN. [ticket: X] : NA Author : bernardo.gomezpalacio@gmail.com Reviewer : ? Testing : ? --- bin/compute-classpath.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index 2a2bb376fd71..2efec362b1df 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -44,6 +44,7 @@ if [ -f "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar ]; then CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/classes" + CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SCALA_VERSION/classes" DEPS_ASSEMBLY_JAR=`ls "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar` CLASSPATH="$CLASSPATH:$DEPS_ASSEMBLY_JAR" From 1342886a396be00eda9449c6d84155dfecf954c8 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 14:46:44 -0700 Subject: [PATCH 02/25] The `spark-class` shell now ignores non jar files in the assembly directory. Why is this change necessary? While developing in Spark I found myself rebuilding either the dependencies assembly or the full spark assembly. I kept running into the case of having both the dep-assembly and full-assembly in the same directory and getting an error when I called either `spark-shell` or `spark-submit`. Quick fix: move either of them as a .bkp file depending on the development work flow you are executing at the moment and enabling the `spark-class` to ignore non-jar files. An other option could be to move the "offending" jar to a different directory but in my opinion keeping them in there is a bit tidier. e.g. ``` ll ./assembly/target/scala-2.10 spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0-deps.jar spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0.jar.bkp ``` [ticket: X] : ? --- bin/spark-class | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/spark-class b/bin/spark-class index 1b0d309cc5b1..ff03f1fda6e9 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -109,8 +109,8 @@ export JAVA_OPTS if [ ! -f "$FWDIR/RELEASE" ]; then # Exit if the user hasn't compiled Spark - num_jars=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep "spark-assembly.*hadoop.*.jar" | wc -l) - jars_list=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep "spark-assembly.*hadoop.*.jar") + num_jars=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep -E "spark-assembly.*hadoop.*.jar$" | wc -l) + jars_list=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep -E "spark-assembly.*hadoop.*.jar$") if [ "$num_jars" -eq "0" ]; then echo "Failed to find Spark assembly in $FWDIR/assembly/target/scala-$SCALA_VERSION/" >&2 echo "You need to build Spark with 'sbt/sbt assembly' before running this program." >&2 From ddf2547aa2aea8155f8d6c0386e2cb37bcf61537 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 14:53:23 -0700 Subject: [PATCH 03/25] The `spark-shell` option `--log-conf` also enables the SPARK_PRINT_LAUNCH_COMMAND . Why is this change necessary? Most likely when enabling the `--log-conf` through the `spark-shell` you are also interested on the full invocation of the java command including the _classpath_ and extended options. e.g. ``` INFO: Base Directory set to /Users/bernardo/work/github/berngp/spark INFO: Spark Master is yarn-client INFO: Spark REPL options -Dspark.logConf=true Spark Command: /Library/Java/JavaVirtualMachines/jdk1.8.0.jdk/Contents/Home/bin/java -cp :/Users/bernardo/work/github/berngp/spark/conf:/Users/bernardo/work/github/berngp/spark/core/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/repl/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/mllib/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/bagel/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/graphx/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/streaming/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/tools/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/sql/catalyst/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/sql/core/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/sql/hive/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/yarn/stable/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/assembly/target/scala-2.10/spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0-deps.jar:/usr/local/Cellar/hadoop/2.2.0/libexec/etc/hadoop -XX:ErrorFile=/tmp/spark-shell-hs_err_pid.log -XX:HeapDumpPath=/tmp/spark-shell-java_pid.hprof -XX:-HeapDumpOnOutOfMemoryError -XX:-PrintGC -XX:-PrintGCDetails -XX:-PrintGCTimeStamps -XX:-PrintTenuringDistribution -XX:-PrintAdaptiveSizePolicy -XX:GCLogFileSize=1024K -XX:-UseGCLogFileRotation -Xloggc:/tmp/spark-shell-gc.log -XX:+UseConcMarkSweepGC -Dspark.cleaner.ttl=10000 -Dspark.driver.host=33.33.33.1 -Dspark.logConf=true -Djava.library.path= -Xms400M -Xmx400M org.apache.spark.repl.Main ``` [ticket: X] : ? --- bin/spark-shell | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/spark-shell b/bin/spark-shell index ea12d256b23a..cc9d1ea20146 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -123,6 +123,7 @@ function set_dm(){ function set_spark_log_conf(){ SPARK_REPL_OPTS="$SPARK_REPL_OPTS -Dspark.logConf=$1" + export SPARK_PRINT_LAUNCH_COMMAND=1 } function set_spark_master(){ From 22045394955992c2c8dfe0e1040c6bb972be6ce4 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 15:15:23 -0700 Subject: [PATCH 04/25] Root is now Spark and qualify the assembly if it was built with YARN. Why is this change necessary? Renamed the SBT "root" project to "spark" to enhance readability. Currently the assembly is qualified with the Hadoop Version but not if YARN has been enabled or not. This change qualifies the assembly such that it is easy to identify if YARN was enabled. e.g ``` ./make-distribution.sh --hadoop 2.3.0 --with-yarn ls -l ./assembly/target/scala-2.10 spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0-yarn.jar ``` vs ``` ./make-distribution.sh --hadoop 2.3.0 ls -l ./assembly/target/scala-2.10 spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0.jar ``` [ticket: X] : ? --- project/SparkBuild.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 09b527c76a5a..f6e8a6ade161 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -52,7 +52,7 @@ object SparkBuild extends Build { val SCALAC_JVM_VERSION = "jvm-1.6" val JAVAC_JVM_VERSION = "1.6" - lazy val root = Project("root", file("."), settings = rootSettings) aggregate(allProjects: _*) + lazy val root = Project("spark", file("."), settings = rootSettings) aggregate(allProjects: _*) lazy val core = Project("core", file("core"), settings = coreSettings) @@ -409,7 +409,7 @@ object SparkBuild extends Build { def catalystSettings = sharedSettings ++ Seq( name := "catalyst", // The mechanics of rewriting expression ids to compare trees in some test cases makes - // assumptions about the the expression ids being contiguious. Running tests in parallel breaks + // assumptions about the the expression ids being contiguous. Running tests in parallel breaks // this non-deterministically. TODO: FIX THIS. parallelExecution in Test := false, libraryDependencies ++= Seq( @@ -516,7 +516,7 @@ object SparkBuild extends Build { libraryDependencies += "net.sf.py4j" % "py4j" % "0.8.1", name := "spark-assembly", assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, - jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }, + jarName in assembly <<= version map { v => s"spark-assembly-${v}-hadoop${hadoopVersion}${if (isYarnEnabled) "-yarn" else ""}.jar" }, jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings From 889bf4ed742ed3d06cb62276ef554f2f37b53ee6 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 17:08:27 -0700 Subject: [PATCH 05/25] Upgrade the Maven Build to YARN 2.3.0. Upgraded to YARN 2.3.0, removed unnecessary `relativePath` values and removed incorrect version for the "org.apache.hadoop:hadoop-client" dependency at yarn/pom.xml. --- pom.xml | 3 ++- yarn/alpha/pom.xml | 1 - yarn/pom.xml | 4 +--- yarn/stable/pom.xml | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index cd204376de5d..0fd3d227a066 100644 --- a/pom.xml +++ b/pom.xml @@ -836,7 +836,8 @@ yarn 2 - 2.2.0 + 2.3.0 + 2.3.0 2.5.0 diff --git a/yarn/alpha/pom.xml b/yarn/alpha/pom.xml index d0aeaceb0d23..1892d2111713 100644 --- a/yarn/alpha/pom.xml +++ b/yarn/alpha/pom.xml @@ -21,7 +21,6 @@ org.apache.spark yarn-parent_2.10 1.0.0-SNAPSHOT - ../pom.xml diff --git a/yarn/pom.xml b/yarn/pom.xml index 3342cb65edcd..f86095896384 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -21,14 +21,13 @@ org.apache.spark spark-parent 1.0.0-SNAPSHOT - ../pom.xml org.apache.spark yarn-parent_2.10 pom Spark Project YARN Parent POM - + org.apache.spark @@ -50,7 +49,6 @@ org.apache.hadoop hadoop-client - ${yarn.version} org.scalatest diff --git a/yarn/stable/pom.xml b/yarn/stable/pom.xml index e7915d12aef6..22ba11461213 100644 --- a/yarn/stable/pom.xml +++ b/yarn/stable/pom.xml @@ -21,7 +21,6 @@ org.apache.spark yarn-parent_2.10 1.0.0-SNAPSHOT - ../pom.xml From f1c7535fe6e97e1d5ebf8adcac01d82c794a01f8 Mon Sep 17 00:00:00 2001 From: witgo Date: Wed, 30 Apr 2014 01:48:01 +0800 Subject: [PATCH 06/25] =?UTF-8?q?Improved=20build=20configuration=20?= =?UTF-8?q?=E2=85=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/building-with-maven.md | 9 ++- pom.xml | 133 ++++++++++++++++++------------------ 2 files changed, 72 insertions(+), 70 deletions(-) diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index a5e530346740..5eb85f05a4d4 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -45,17 +45,20 @@ For Apache Hadoop versions 1.x, Cloudera CDH MRv1, and other Hadoop versions wit For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with YARN, you can enable the "yarn-alpha" or "yarn" profile and set the "hadoop.version", "yarn.version" property. Note that Hadoop 0.23.X requires a special `-Phadoop-0.23` profile: # Apache Hadoop 2.0.5-alpha - $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -Dyarn.version=2.0.5-alpha -DskipTests clean package + $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -DskipTests clean package # Cloudera CDH 4.2.0 with MapReduce v2 - $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -Dyarn.version=2.0.0-cdh4.2.0 -DskipTests clean package + $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -DskipTests clean package # Apache Hadoop 2.2.X (e.g. 2.2.0 as below) and newer - $ mvn -Pyarn -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -DskipTests clean package + $ mvn -Pyarn -Dhadoop.version=2.2.0 -DskipTests clean package # Apache Hadoop 0.23.x $ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -Dyarn.version=0.23.7 -DskipTests clean package + # Different versions of HDFS vs YARN. + $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version= 0.23.7 -DskipTests clean package + ## Spark Tests in Maven ## Tests are run by default via the [ScalaTest Maven plugin](http://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin). Some of the require Spark to be packaged first, so always run `mvn package` with `-DskipTests` the first time. You can then run the tests with `mvn -Dhadoop.version=... test`. diff --git a/pom.xml b/pom.xml index 560bd364ff91..73de1cc385c5 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,8 @@ ~ limitations under the License. --> - + 4.0.0 org.apache @@ -119,7 +120,7 @@ 1.2.17 1.0.4 2.4.1 - 0.23.7 + ${hadoop.version} 0.94.6 0.12.0 1.3.2 @@ -135,7 +136,8 @@ - maven-repo + maven-repo + Maven Repository http://repo.maven.apache.org/maven2 @@ -558,64 +560,7 @@ jets3t 0.7.1 - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - org.codehaus.jackson @@ -850,12 +795,6 @@ yarn - - - org.apache.avro - avro - - @@ -896,13 +835,73 @@ 2 2.3.0 - 2.3.0 2.5.0 yarn + + + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + From 8540e83420a7d11394501f88872860d32ebc3f1b Mon Sep 17 00:00:00 2001 From: witgo Date: Wed, 30 Apr 2014 09:37:06 +0800 Subject: [PATCH 07/25] review commit --- docs/building-with-maven.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index 5eb85f05a4d4..e447dfea3bac 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -56,8 +56,8 @@ For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with # Apache Hadoop 0.23.x $ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -Dyarn.version=0.23.7 -DskipTests clean package - # Different versions of HDFS vs YARN. - $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version= 0.23.7 -DskipTests clean package + # Different versions of HDFS and YARN. + $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version=0.23.7 -DskipTests clean package ## Spark Tests in Maven ## From c4c6e45ff134663196e07fbd3a812af57b8ea95c Mon Sep 17 00:00:00 2001 From: witgo Date: Wed, 30 Apr 2014 23:23:08 +0800 Subject: [PATCH 08/25] review commit --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 45b4a753ef07..f609343dd1ba 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -569,7 +569,7 @@ object SparkBuild extends Build { libraryDependencies += "net.sf.py4j" % "py4j" % "0.8.1", name := "spark-assembly", assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, - jarName in assembly <<= version map { v => s"spark-assembly-${v}-hadoop${hadoopVersion}${if (isYarnEnabled) "-yarn" else ""}.jar" }, + jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" } jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings From e1a7e0020f3c42b9e7dae07cabd072f0dd38d463 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 14:34:57 +0800 Subject: [PATCH 09/25] improve travis tests coverage --- .gitignore | 1 + .jvmopts | 5 +++++ .travis.yml | 28 +++++++++++++++++++++------- core/pom.xml | 29 ----------------------------- pom.xml | 5 +++++ repl/pom.xml | 29 ----------------------------- yarn/pom.xml | 29 ----------------------------- 7 files changed, 32 insertions(+), 94 deletions(-) create mode 100644 .jvmopts diff --git a/.gitignore b/.gitignore index 857e9feb953b..a509bfbb495e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ conf/spark-env.sh conf/streaming-env.sh conf/log4j.properties conf/spark-defaults.conf +conf/*.xml docs/_site docs/api target/ diff --git a/.jvmopts b/.jvmopts new file mode 100644 index 000000000000..86db0ab9b6dd --- /dev/null +++ b/.jvmopts @@ -0,0 +1,5 @@ +-Xmx3g +-Xss2M +-XX:+CMSClassUnloadingEnabled +-XX:MaxPermSize=512M +-XX:ReservedCodeCacheSize=512m \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 8ebd0d68429f..4895faed78d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,18 +15,32 @@ language: scala scala: - - "2.10.3" + - "2.10.4" jdk: + - oraclejdk8 - oraclejdk7 - env: - matrix: - - TEST="scalastyle assembly/assembly" - - TEST="catalyst/test sql/test streaming/test mllib/test graphx/test bagel/test" - - TEST=hive/test + - openjdk6 cache: directories: - $HOME/.m2 - $HOME/.ivy2 - $HOME/.sbt + env: + matrix: + - SPARK_HADOOP_VERSION=1.2.1 + - SPARK_HADOOP_VERSION=2.4.0 + before_script: + - sudo apt-get install libgfortran3 script: - - "sbt ++$TRAVIS_SCALA_VERSION $TEST" + # TODO: Cannot get hive/test to pass + > + if [[ $SPARK_HADOOP_VERSION = '2.4.0' ]]; then + mvn clean package -DskipTests -Pyarn -Dhadoop.version=$SPARK_HADOOP_VERSION -Dyarn.version=$SPARK_HADOOP_VERSION + num_jars=$(ls "$FWDIR"/assembly/target/scala-2.10/ | grep "spark-assembly.*hadoop.*.jar" | wc -l) + if [ "$num_jars" -eq "0" ]; then + exit -1 + fi + mvn test -Pyarn -Dhadoop.version=$SPARK_HADOOP_VERSION -Dyarn.version=$SPARK_HADOOP_VERSION -am -pl mllib -pl bagel -pl sql/catalyst -pl yarn + else + SPARK_HADOOP_VERSION=$SPARK_HADOOP_VERSION sbt ++$TRAVIS_SCALA_VERSION assembly/assembly sql/test graphx/test streaming/test + fi \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml index 822b5b1dd7cc..1f71bb95f857 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -254,35 +254,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - org.apache.maven.plugins - maven-antrun-plugin - - - test - - run - - - true - - - - - - - - - - - - - - - - - - org.scalatest scalatest-maven-plugin diff --git a/pom.xml b/pom.xml index 73de1cc385c5..e1e0bce3929d 100644 --- a/pom.xml +++ b/pom.xml @@ -682,6 +682,11 @@ ${project.build.directory}/SparkTestSuite.txt -Xmx3g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=512m + + ${basedir}/.. + 1 + ${spark.classpath} + diff --git a/repl/pom.xml b/repl/pom.xml index b761a176ce25..8539a9394ddd 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -92,35 +92,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - org.apache.maven.plugins - maven-antrun-plugin - - - test - - run - - - true - - - - - - - - - - - - - - - - - - org.scalatest scalatest-maven-plugin diff --git a/yarn/pom.xml b/yarn/pom.xml index a10436c07d91..35b5090a102a 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -112,35 +112,6 @@ - - org.apache.maven.plugins - maven-antrun-plugin - - - test - - run - - - true - - - - - - - - - - - - - - - - - - org.scalatest scalatest-maven-plugin From effe79cd8c018c31be31c53260fad59fa50a77a9 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 14:46:04 +0800 Subject: [PATCH 10/25] missing "," --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f609343dd1ba..03e25acf6f1b 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -569,7 +569,7 @@ object SparkBuild extends Build { libraryDependencies += "net.sf.py4j" % "py4j" % "0.8.1", name := "spark-assembly", assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, - jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" } + jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }, jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings From 9ea1af926c3ccf36c59e13c103ec6052c032f77e Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 15:04:53 +0800 Subject: [PATCH 11/25] add the dependency of commons-lang --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index e1e0bce3929d..76e0e92a4bc1 100644 --- a/pom.xml +++ b/pom.xml @@ -373,6 +373,11 @@ commons-net 2.2 + + commons-lang + commons-lang + 2.5 + io.netty netty-all From 0ed124dc0e453a0a59d3c387651be970859a9a0a Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 16:07:45 +0800 Subject: [PATCH 12/25] SPARK-1693: Most of the tests throw a java.lang.SecurityException when spark built for hadoop 2.3.0 , 2.4.0 --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index 76e0e92a4bc1..1a7f590cc333 100644 --- a/pom.xml +++ b/pom.xml @@ -888,6 +888,10 @@ org.jboss.netty netty + + javax.servlet + servlet-api + @@ -908,6 +912,10 @@ org.jboss.netty netty + + javax.servlet + servlet-api + From 03b136fe1ad64ee12ffe90bacfe715a2ad87b2c2 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 16:27:11 +0800 Subject: [PATCH 13/25] revert .travis.yml --- .jvmopts | 5 ----- .travis.yml | 28 +++++++--------------------- 2 files changed, 7 insertions(+), 26 deletions(-) delete mode 100644 .jvmopts diff --git a/.jvmopts b/.jvmopts deleted file mode 100644 index 86db0ab9b6dd..000000000000 --- a/.jvmopts +++ /dev/null @@ -1,5 +0,0 @@ --Xmx3g --Xss2M --XX:+CMSClassUnloadingEnabled --XX:MaxPermSize=512M --XX:ReservedCodeCacheSize=512m \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 4895faed78d2..a37e958c6809 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,32 +15,18 @@ language: scala scala: - - "2.10.4" + - "2.10.3" jdk: - - oraclejdk8 - oraclejdk7 - - openjdk6 + env: + matrix: + - TEST="scalastyle assembly/assembly" + - TEST="catalyst/test sql/test streaming/test mllib/test graphx/test bagel/test" + - TEST=hive/test cache: directories: - $HOME/.m2 - $HOME/.ivy2 - $HOME/.sbt - env: - matrix: - - SPARK_HADOOP_VERSION=1.2.1 - - SPARK_HADOOP_VERSION=2.4.0 - before_script: - - sudo apt-get install libgfortran3 script: - # TODO: Cannot get hive/test to pass - > - if [[ $SPARK_HADOOP_VERSION = '2.4.0' ]]; then - mvn clean package -DskipTests -Pyarn -Dhadoop.version=$SPARK_HADOOP_VERSION -Dyarn.version=$SPARK_HADOOP_VERSION - num_jars=$(ls "$FWDIR"/assembly/target/scala-2.10/ | grep "spark-assembly.*hadoop.*.jar" | wc -l) - if [ "$num_jars" -eq "0" ]; then - exit -1 - fi - mvn test -Pyarn -Dhadoop.version=$SPARK_HADOOP_VERSION -Dyarn.version=$SPARK_HADOOP_VERSION -am -pl mllib -pl bagel -pl sql/catalyst -pl yarn - else - SPARK_HADOOP_VERSION=$SPARK_HADOOP_VERSION sbt ++$TRAVIS_SCALA_VERSION assembly/assembly sql/test graphx/test streaming/test - fi \ No newline at end of file + - "sbt ++$TRAVIS_SCALA_VERSION $TEST" \ No newline at end of file From d3488c6e316657ca1436b8cc148cfb9102747658 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 17:19:15 +0800 Subject: [PATCH 14/25] Add the missing yarn dependencies --- pom.xml | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/pom.xml b/pom.xml index 1a7f590cc333..74567d723f32 100644 --- a/pom.xml +++ b/pom.xml @@ -805,6 +805,76 @@ yarn + + + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + + + From 779ae5d71992da80a20259978b4cd900110c8f30 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 22:58:12 +0800 Subject: [PATCH 15/25] Fix SPARK-1693: Dependent on multiple versions of servlet-api jars lead to throw an SecurityException when Spark built for hadoop 2.3.0 , 2.4.0 --- bagel/pom.xml | 4 ++++ core/pom.xml | 4 ++++ examples/pom.xml | 4 ++++ graphx/pom.xml | 4 ++++ mllib/pom.xml | 4 ++++ pom.xml | 13 +++++++++++++ project/SparkBuild.scala | 22 ++++++++++++---------- repl/pom.xml | 4 ++++ streaming/pom.xml | 4 ++++ 9 files changed, 53 insertions(+), 10 deletions(-) diff --git a/bagel/pom.xml b/bagel/pom.xml index 355f437c5b16..adb691648213 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -41,6 +41,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.scalatest scalatest_${scala.binary.version} diff --git a/core/pom.xml b/core/pom.xml index 1f71bb95f857..54b53f9d4260 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -65,6 +65,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + com.google.guava guava diff --git a/examples/pom.xml b/examples/pom.xml index e1fc149d87f1..f6b3f8150b96 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -120,6 +120,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + com.twitter algebird-core_${scala.binary.version} diff --git a/graphx/pom.xml b/graphx/pom.xml index dc108d2fe7fb..f8abc7f3c8ac 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -46,6 +46,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.scalatest scalatest_${scala.binary.version} diff --git a/mllib/pom.xml b/mllib/pom.xml index cdd33dbb7970..947d0d919d64 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -41,6 +41,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.jblas jblas diff --git a/pom.xml b/pom.xml index 74567d723f32..df40e1808991 100644 --- a/pom.xml +++ b/pom.xml @@ -215,6 +215,19 @@ org.eclipse.jetty jetty-server ${jetty.version} + + + org.eclipse.jetty.orbit + javax.servlet + + + + + + javax.servlet + javax.servlet-api + 3.0.1 com.google.guava diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 03e25acf6f1b..1b52d596c449 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -266,16 +266,17 @@ object SparkBuild extends Build { */ libraryDependencies ++= Seq( - "io.netty" % "netty-all" % "4.0.17.Final", - "org.eclipse.jetty" % "jetty-server" % jettyVersion, - "org.eclipse.jetty" % "jetty-util" % jettyVersion, - "org.eclipse.jetty" % "jetty-plus" % jettyVersion, - "org.eclipse.jetty" % "jetty-security" % jettyVersion, - "org.scalatest" %% "scalatest" % "1.9.1" % "test", - "org.scalacheck" %% "scalacheck" % "1.10.0" % "test", - "com.novocode" % "junit-interface" % "0.10" % "test", - "org.easymock" % "easymock" % "3.1" % "test", - "org.mockito" % "mockito-all" % "1.8.5" % "test" + "io.netty" % "netty-all" % "4.0.17.Final", + "org.eclipse.jetty" % "jetty-server" % jettyVersion excludeAll(excludeJettyServlet), + "javax.servlet" % "javax.servlet-api" % "3.0.1", + "org.eclipse.jetty" % "jetty-util" % jettyVersion, + "org.eclipse.jetty" % "jetty-plus" % jettyVersion, + "org.eclipse.jetty" % "jetty-security" % jettyVersion, + "org.scalatest" %% "scalatest" % "1.9.1" % "test", + "org.scalacheck" %% "scalacheck" % "1.10.0" % "test", + "com.novocode" % "junit-interface" % "0.10" % "test", + "org.easymock" % "easymock" % "3.1" % "test", + "org.mockito" % "mockito-all" % "1.8.5" % "test" ), testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), @@ -315,6 +316,7 @@ object SparkBuild extends Build { val excludeFastutil = ExclusionRule(organization = "it.unimi.dsi") val excludeJruby = ExclusionRule(organization = "org.jruby") val excludeThrift = ExclusionRule(organization = "org.apache.thrift") + val excludeJettyServlet= ExclusionRule(organization = "org.eclipse.jetty.orbit") def sparkPreviousArtifact(id: String, organization: String = "org.apache.spark", version: String = "0.9.0-incubating", crossVersion: String = "2.10"): Option[sbt.ModuleID] = { diff --git a/repl/pom.xml b/repl/pom.xml index 8539a9394ddd..e9d812ee8030 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -58,6 +58,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.scala-lang scala-compiler diff --git a/streaming/pom.xml b/streaming/pom.xml index 6435224a1467..ecbafd833418 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -41,6 +41,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.scala-lang scala-library From 27bd4268a7d0c7300182f43d4edc59ad8154aa0c Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 23:01:42 +0800 Subject: [PATCH 16/25] review commit --- yarn/alpha/pom.xml | 1 + yarn/pom.xml | 1 + yarn/stable/pom.xml | 1 + 3 files changed, 3 insertions(+) diff --git a/yarn/alpha/pom.xml b/yarn/alpha/pom.xml index 14e5bfc518b0..e076ca1d44b9 100644 --- a/yarn/alpha/pom.xml +++ b/yarn/alpha/pom.xml @@ -21,6 +21,7 @@ org.apache.spark yarn-parent_2.10 1.0.0-SNAPSHOT + ../pom.xml org.apache.spark diff --git a/yarn/pom.xml b/yarn/pom.xml index 35b5090a102a..1a2729466acd 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -21,6 +21,7 @@ org.apache.spark spark-parent 1.0.0-SNAPSHOT + ../pom.xml org.apache.spark diff --git a/yarn/stable/pom.xml b/yarn/stable/pom.xml index 99a04c2ab9cc..0780f251b595 100644 --- a/yarn/stable/pom.xml +++ b/yarn/stable/pom.xml @@ -21,6 +21,7 @@ org.apache.spark yarn-parent_2.10 1.0.0-SNAPSHOT + ../pom.xml org.apache.spark From 54a86b03b5b0b3f90f2b3c78cb9456a0ec81976b Mon Sep 17 00:00:00 2001 From: witgo Date: Fri, 2 May 2014 14:22:36 +0800 Subject: [PATCH 17/25] review commit --- pom.xml | 3 +-- repl/pom.xml | 1 - yarn/pom.xml | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index df40e1808991..e9f51abfc85d 100644 --- a/pom.xml +++ b/pom.xml @@ -701,9 +701,8 @@ -Xmx3g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=512m - ${basedir}/.. + ${session.executionRootDirectory} 1 - ${spark.classpath} diff --git a/repl/pom.xml b/repl/pom.xml index e9d812ee8030..c686b2f978ef 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -102,7 +102,6 @@ ${basedir}/.. - 1 diff --git a/yarn/pom.xml b/yarn/pom.xml index 1a2729466acd..b8714b54502e 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -119,7 +119,6 @@ ${basedir}/../.. - 1 ${spark.classpath} From 882e35d354e98b8a416583038b72a49d0a331d8e Mon Sep 17 00:00:00 2001 From: witgo Date: Fri, 2 May 2014 14:24:05 +0800 Subject: [PATCH 18/25] review commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e9f51abfc85d..ad9a2ca70e9e 100644 --- a/pom.xml +++ b/pom.xml @@ -926,7 +926,7 @@ yarn 2 - 2.3.0 + 2.2.0 2.5.0 From 31451dfe66e14f6ccf7daa32287190c7ef7780ca Mon Sep 17 00:00:00 2001 From: witgo Date: Sat, 3 May 2014 11:15:32 +0800 Subject: [PATCH 19/25] Compile hive optional --- pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ad9a2ca70e9e..71def98165f0 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,6 @@ streaming sql/catalyst sql/core - sql/hive repl assembly external/twitter @@ -1049,6 +1048,12 @@ + + hive + + sql/hive + + From 5fb961fdd7b601c937074960641ef17fbd08a7b5 Mon Sep 17 00:00:00 2001 From: witgo Date: Sat, 3 May 2014 15:54:19 +0800 Subject: [PATCH 20/25] revert exclusion org.eclipse.jetty.orbit:javax.servlet --- bagel/pom.xml | 4 ---- core/pom.xml | 4 ---- examples/pom.xml | 4 ---- graphx/pom.xml | 4 ---- mllib/pom.xml | 4 ---- pom.xml | 15 ++------------- repl/pom.xml | 4 ---- streaming/pom.xml | 4 ---- 8 files changed, 2 insertions(+), 41 deletions(-) diff --git a/bagel/pom.xml b/bagel/pom.xml index adb691648213..355f437c5b16 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -41,10 +41,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.scalatest scalatest_${scala.binary.version} diff --git a/core/pom.xml b/core/pom.xml index 54b53f9d4260..1f71bb95f857 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -65,10 +65,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - com.google.guava guava diff --git a/examples/pom.xml b/examples/pom.xml index f6b3f8150b96..e1fc149d87f1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -120,10 +120,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - com.twitter algebird-core_${scala.binary.version} diff --git a/graphx/pom.xml b/graphx/pom.xml index f8abc7f3c8ac..dc108d2fe7fb 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -46,10 +46,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.scalatest scalatest_${scala.binary.version} diff --git a/mllib/pom.xml b/mllib/pom.xml index 947d0d919d64..cdd33dbb7970 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -41,10 +41,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.jblas jblas diff --git a/pom.xml b/pom.xml index 71def98165f0..e79972cbb42c 100644 --- a/pom.xml +++ b/pom.xml @@ -214,19 +214,6 @@ org.eclipse.jetty jetty-server ${jetty.version} - - - org.eclipse.jetty.orbit - javax.servlet - - - - - - javax.servlet - javax.servlet-api - 3.0.1 com.google.guava @@ -854,6 +841,8 @@ org.jboss.netty netty + javax.servlet servlet-api diff --git a/repl/pom.xml b/repl/pom.xml index c686b2f978ef..bcdb24b040cc 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -58,10 +58,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.scala-lang scala-compiler diff --git a/streaming/pom.xml b/streaming/pom.xml index ecbafd833418..6435224a1467 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -41,10 +41,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.scala-lang scala-library From a5ff7d13f523a6494842f61982f706ada4f9a63a Mon Sep 17 00:00:00 2001 From: witgo Date: Sat, 3 May 2014 18:50:53 +0800 Subject: [PATCH 21/25] revert exclusion org.eclipse.jetty.orbit:javax.servlet --- .travis.yml | 2 +- project/SparkBuild.scala | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a37e958c6809..8ebd0d68429f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,4 +29,4 @@ - $HOME/.ivy2 - $HOME/.sbt script: - - "sbt ++$TRAVIS_SCALA_VERSION $TEST" \ No newline at end of file + - "sbt ++$TRAVIS_SCALA_VERSION $TEST" diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 1b52d596c449..b537bc7ed579 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -267,8 +267,7 @@ object SparkBuild extends Build { libraryDependencies ++= Seq( "io.netty" % "netty-all" % "4.0.17.Final", - "org.eclipse.jetty" % "jetty-server" % jettyVersion excludeAll(excludeJettyServlet), - "javax.servlet" % "javax.servlet-api" % "3.0.1", + "org.eclipse.jetty" % "jetty-server" % jettyVersion, "org.eclipse.jetty" % "jetty-util" % jettyVersion, "org.eclipse.jetty" % "jetty-plus" % jettyVersion, "org.eclipse.jetty" % "jetty-security" % jettyVersion, @@ -316,7 +315,6 @@ object SparkBuild extends Build { val excludeFastutil = ExclusionRule(organization = "it.unimi.dsi") val excludeJruby = ExclusionRule(organization = "org.jruby") val excludeThrift = ExclusionRule(organization = "org.apache.thrift") - val excludeJettyServlet= ExclusionRule(organization = "org.eclipse.jetty.orbit") def sparkPreviousArtifact(id: String, organization: String = "org.apache.spark", version: String = "0.9.0-incubating", crossVersion: String = "2.10"): Option[sbt.ModuleID] = { From 427d499247b0e3286270da322b547a61f8fe0a89 Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 12 May 2014 10:31:22 +0800 Subject: [PATCH 22/25] merge master --- pom.xml | 224 +++++++++++++++++++++----------------------------------- 1 file changed, 84 insertions(+), 140 deletions(-) diff --git a/pom.xml b/pom.xml index 38b7a60ad57c..97b8eca44a9b 100644 --- a/pom.xml +++ b/pom.xml @@ -546,6 +546,90 @@ + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + + + org.apache.hadoop + hadoop-yarn-server-web-proxy + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + org.codehaus.jackson @@ -844,76 +928,6 @@ yarn - - - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - - - @@ -921,76 +935,6 @@ yarn - - - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - - - From 4cc0c902a923f31da10a5b110a9bb99a0b717c5b Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 12 May 2014 13:35:02 +0800 Subject: [PATCH 23/25] revert profile hive --- pom.xml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 97b8eca44a9b..34dc10b527a9 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,7 @@ streaming sql/catalyst sql/core + sql/hive repl assembly external/twitter @@ -880,13 +881,6 @@ - - hive - - sql/hive - - - From 4277fedc6ff55c6cca41e4f5d2e6f88f346977af Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 12 May 2014 17:01:02 +0800 Subject: [PATCH 24/25] review commit --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index 34dc10b527a9..e02af0201f79 100644 --- a/pom.xml +++ b/pom.xml @@ -355,11 +355,6 @@ commons-net 2.2 - - commons-lang - commons-lang - 2.5 - io.netty netty-all From 31c64093abd990ec812cad5afb902d40cc49737b Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 12 May 2014 17:01:54 +0800 Subject: [PATCH 25/25] review commit --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index a471c28d6f83..ad72588b472d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ conf/spark-env.sh conf/streaming-env.sh conf/log4j.properties conf/spark-defaults.conf -conf/*.xml docs/_site docs/api target/