Skip to content

Commit 1bbbfb0

Browse files
MLnickMLnick
committed
Clean up SparkBuild from merge
1 parent a67dfad commit 1bbbfb0

File tree

1 file changed

+143
-84
lines changed

1 file changed

+143
-84
lines changed

‎project/SparkBuild.scala‎

Lines changed: 143 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ object SparkBuild extends Build {
189189
"org.apache.spark.network",
190190
"org.apache.spark.deploy",
191191
"org.apache.spark.util.collection"
192-
).mkString(":")),
192+
).mkString(":")),
193193

194194
// Only allow one test at a time, even across projects, since they run in the same JVM
195195
concurrentRestrictions in Global += Tags.limit(Tags.Test, 1),
@@ -218,33 +218,33 @@ object SparkBuild extends Build {
218218
<artifactId>apache</artifactId>
219219
<version>13</version>
220220
</parent>
221-
<url>http://spark.apache.org/</url>
222-
<licenses>
223-
<license>
224-
<name>Apache 2.0 License</name>
225-
<url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
226-
<distribution>repo</distribution>
227-
</license>
228-
</licenses>
229-
<scm>
230-
<connection>scm:git:git@github.com:apache/spark.git</connection>
231-
<url>scm:git:git@github.com:apache/spark.git</url>
232-
</scm>
233-
<developers>
234-
<developer>
235-
<id>matei</id>
236-
<name>Matei Zaharia</name>
237-
<email>matei.zaharia@gmail.com</email>
238-
<url>http://www.cs.berkeley.edu/~matei</url>
239-
<organization>Apache Software Foundation</organization>
240-
<organizationUrl>http://spark.apache.org</organizationUrl>
241-
</developer>
242-
</developers>
243-
<issueManagement>
244-
<system>JIRA</system>
245-
<url>https://spark-project.atlassian.net/browse/SPARK</url>
246-
</issueManagement>
247-
),
221+
<url>http://spark.apache.org/</url>
222+
<licenses>
223+
<license>
224+
<name>Apache 2.0 License</name>
225+
<url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
226+
<distribution>repo</distribution>
227+
</license>
228+
</licenses>
229+
<scm>
230+
<connection>scm:git:git@github.com:apache/spark.git</connection>
231+
<url>scm:git:git@github.com:apache/spark.git</url>
232+
</scm>
233+
<developers>
234+
<developer>
235+
<id>matei</id>
236+
<name>Matei Zaharia</name>
237+
<email>matei.zaharia@gmail.com</email>
238+
<url>http://www.cs.berkeley.edu/~matei</url>
239+
<organization>Apache Software Foundation</organization>
240+
<organizationUrl>http://spark.apache.org</organizationUrl>
241+
</developer>
242+
</developers>
243+
<issueManagement>
244+
<system>JIRA</system>
245+
<url>https://spark-project.atlassian.net/browse/SPARK</url>
246+
</issueManagement>
247+
),
248248

249249
/*
250250
publishTo <<= version { (v: String) =>
@@ -258,18 +258,18 @@ object SparkBuild extends Build {
258258
*/
259259

260260
libraryDependencies ++= Seq(
261-
"io.netty" % "netty-all" % "4.0.17.Final",
262-
"org.eclipse.jetty" % "jetty-server" % jettyVersion,
263-
"org.eclipse.jetty" % "jetty-util" % jettyVersion,
264-
"org.eclipse.jetty" % "jetty-plus" % jettyVersion,
265-
"org.eclipse.jetty" % "jetty-security" % jettyVersion,
266-
/** Workaround for SPARK-959. Dependency used by org.eclipse.jetty. Fixed in ivy 2.3.0. */
267-
"org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" artifacts Artifact("javax.servlet", "jar", "jar"),
268-
"org.scalatest" %% "scalatest" % "1.9.1" % "test",
269-
"org.scalacheck" %% "scalacheck" % "1.10.0" % "test",
270-
"com.novocode" % "junit-interface" % "0.10" % "test",
271-
"org.easymock" % "easymock" % "3.1" % "test",
272-
"org.mockito" % "mockito-all" % "1.8.5" % "test"
261+
"io.netty" % "netty-all" % "4.0.17.Final",
262+
"org.eclipse.jetty" % "jetty-server" % jettyVersion,
263+
"org.eclipse.jetty" % "jetty-util" % jettyVersion,
264+
"org.eclipse.jetty" % "jetty-plus" % jettyVersion,
265+
"org.eclipse.jetty" % "jetty-security" % jettyVersion,
266+
/** Workaround for SPARK-959. Dependency used by org.eclipse.jetty. Fixed in ivy 2.3.0. */
267+
"org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" artifacts Artifact("javax.servlet", "jar", "jar"),
268+
"org.scalatest" %% "scalatest" % "1.9.1" % "test",
269+
"org.scalacheck" %% "scalacheck" % "1.10.0" % "test",
270+
"com.novocode" % "junit-interface" % "0.10" % "test",
271+
"org.easymock" % "easymock" % "3.1" % "test",
272+
"org.mockito" % "mockito-all" % "1.8.5" % "test"
273273
),
274274

275275
testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
@@ -309,46 +309,46 @@ object SparkBuild extends Build {
309309

310310

311311
def sparkPreviousArtifact(id: String, organization: String = "org.apache.spark",
312-
version: String = "0.9.0-incubating", crossVersion: String = "2.10"): Option[sbt.ModuleID] = {
312+
version: String = "0.9.0-incubating", crossVersion: String = "2.10"): Option[sbt.ModuleID] = {
313313
val fullId = if (crossVersion.isEmpty) id else id + "_" + crossVersion
314314
Some(organization % fullId % version) // the artifact to compare binary compatibility with
315315
}
316316

317317
def coreSettings = sharedSettings ++ Seq(
318318
name := "spark-core",
319319
libraryDependencies ++= Seq(
320-
"com.google.guava" % "guava" % "14.0.1",
321-
"com.google.code.findbugs" % "jsr305" % "1.3.9",
322-
"log4j" % "log4j" % "1.2.17",
323-
"org.slf4j" % "slf4j-api" % slf4jVersion,
324-
"org.slf4j" % "slf4j-log4j12" % slf4jVersion,
325-
"org.slf4j" % "jul-to-slf4j" % slf4jVersion,
326-
"org.slf4j" % "jcl-over-slf4j" % slf4jVersion,
327-
"commons-daemon" % "commons-daemon" % "1.0.10", // workaround for bug HADOOP-9407
328-
"com.ning" % "compress-lzf" % "1.0.0",
329-
"org.xerial.snappy" % "snappy-java" % "1.0.5",
330-
"org.spark-project.akka" %% "akka-remote" % akkaVersion excludeAll(excludeNetty),
331-
"org.spark-project.akka" %% "akka-slf4j" % akkaVersion excludeAll(excludeNetty),
332-
"org.spark-project.akka" %% "akka-testkit" % akkaVersion % "test",
333-
"org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap),
334-
"it.unimi.dsi" % "fastutil" % "6.4.4",
335-
"colt" % "colt" % "1.2.0",
336-
"org.apache.mesos" % "mesos" % "0.17.0",
337-
"commons-net" % "commons-net" % "2.2",
338-
"net.java.dev.jets3t" % "jets3t" % "0.7.1" excludeAll(excludeCommonsLogging),
339-
"org.apache.derby" % "derby" % "10.4.2.0" % "test",
340-
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm),
341-
"org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty),
342-
"com.codahale.metrics" % "metrics-core" % codahaleMetricsVersion,
343-
"com.codahale.metrics" % "metrics-jvm" % codahaleMetricsVersion,
344-
"com.codahale.metrics" % "metrics-json" % codahaleMetricsVersion,
345-
"com.codahale.metrics" % "metrics-graphite" % codahaleMetricsVersion,
346-
"com.twitter" %% "chill" % chillVersion excludeAll(excludeAsm),
347-
"com.twitter" % "chill-java" % chillVersion excludeAll(excludeAsm),
348-
"org.tachyonproject" % "tachyon" % "0.4.1-thrift" excludeAll(excludeHadoop, excludeCurator, excludeEclipseJetty, excludePowermock),
349-
"com.clearspring.analytics" % "stream" % "2.5.1",
350-
"org.msgpack" %% "msgpack-scala" % "0.6.8"
351-
),
320+
"com.google.guava" % "guava" % "14.0.1",
321+
"com.google.code.findbugs" % "jsr305" % "1.3.9",
322+
"log4j" % "log4j" % "1.2.17",
323+
"org.slf4j" % "slf4j-api" % slf4jVersion,
324+
"org.slf4j" % "slf4j-log4j12" % slf4jVersion,
325+
"org.slf4j" % "jul-to-slf4j" % slf4jVersion,
326+
"org.slf4j" % "jcl-over-slf4j" % slf4jVersion,
327+
"commons-daemon" % "commons-daemon" % "1.0.10", // workaround for bug HADOOP-9407
328+
"com.ning" % "compress-lzf" % "1.0.0",
329+
"org.xerial.snappy" % "snappy-java" % "1.0.5",
330+
"org.spark-project.akka" %% "akka-remote" % akkaVersion excludeAll(excludeNetty),
331+
"org.spark-project.akka" %% "akka-slf4j" % akkaVersion excludeAll(excludeNetty),
332+
"org.spark-project.akka" %% "akka-testkit" % akkaVersion % "test",
333+
"org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap),
334+
"it.unimi.dsi" % "fastutil" % "6.4.4",
335+
"colt" % "colt" % "1.2.0",
336+
"org.apache.mesos" % "mesos" % "0.17.0",
337+
"commons-net" % "commons-net" % "2.2",
338+
"net.java.dev.jets3t" % "jets3t" % "0.7.1" excludeAll(excludeCommonsLogging),
339+
"org.apache.derby" % "derby" % "10.4.2.0" % "test",
340+
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm),
341+
"org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty),
342+
"com.codahale.metrics" % "metrics-core" % codahaleMetricsVersion,
343+
"com.codahale.metrics" % "metrics-jvm" % codahaleMetricsVersion,
344+
"com.codahale.metrics" % "metrics-json" % codahaleMetricsVersion,
345+
"com.codahale.metrics" % "metrics-graphite" % codahaleMetricsVersion,
346+
"com.twitter" %% "chill" % chillVersion excludeAll(excludeAsm),
347+
"com.twitter" % "chill-java" % chillVersion excludeAll(excludeAsm),
348+
"org.tachyonproject" % "tachyon" % "0.4.1-thrift" excludeAll(excludeHadoop, excludeCurator, excludeEclipseJetty, excludePowermock),
349+
"com.clearspring.analytics" % "stream" % "2.5.1",
350+
"org.msgpack" %% "msgpack-scala" % "0.6.8"
351+
),
352352
libraryDependencies ++= maybeAvro
353353
)
354354

@@ -358,9 +358,9 @@ object SparkBuild extends Build {
358358

359359
def replSettings = sharedSettings ++ Seq(
360360
name := "spark-repl",
361-
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-compiler" % v ),
362-
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "jline" % v ),
363-
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-reflect" % v )
361+
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-compiler" % v ),
362+
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "jline" % v ),
363+
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-reflect" % v )
364364
)
365365

366366
def examplesSettings = sharedSettings ++ Seq(
@@ -394,33 +394,87 @@ object SparkBuild extends Build {
394394
)
395395

396396
def bagelSettings = sharedSettings ++ Seq(
397-
name := "spark-bagel"
397+
name := "spark-bagel",
398+
previousArtifact := sparkPreviousArtifact("spark-bagel")
398399
)
399400

400401
def mllibSettings = sharedSettings ++ Seq(
401402
name := "spark-mllib",
403+
previousArtifact := sparkPreviousArtifact("spark-mllib"),
402404
libraryDependencies ++= Seq(
403-
"org.jblas" % "jblas" % "1.2.3"
405+
"org.jblas" % "jblas" % jblasVersion,
406+
"org.scalanlp" %% "breeze" % "0.7"
404407
)
405408
)
406409

407-
def streamingSettings = sharedSettings ++ Seq(
408-
name := "spark-streaming",
410+
def catalystSettings = sharedSettings ++ Seq(
411+
name := "catalyst",
412+
// The mechanics of rewriting expression ids to compare trees in some test cases makes
413+
// assumptions about the the expression ids being contiguious. Running tests in parallel breaks
414+
// this non-deterministically. TODO: FIX THIS.
415+
parallelExecution in Test := false,
409416
libraryDependencies ++= Seq(
410-
"commons-io" % "commons-io" % "2.4"
417+
"org.scalatest" %% "scalatest" % "1.9.1" % "test",
418+
"com.typesafe" %% "scalalogging-slf4j" % "1.0.1"
411419
)
412420
)
413421

422+
def sqlCoreSettings = sharedSettings ++ Seq(
423+
name := "spark-sql",
424+
libraryDependencies ++= Seq(
425+
"com.twitter" % "parquet-column" % parquetVersion,
426+
"com.twitter" % "parquet-hadoop" % parquetVersion
427+
)
428+
)
429+
430+
// Since we don't include hive in the main assembly this project also acts as an alternative
431+
// assembly jar.
432+
def hiveSettings = sharedSettings ++ Seq(
433+
name := "spark-hive",
434+
javaOptions += "-XX:MaxPermSize=1g",
435+
libraryDependencies ++= Seq(
436+
"org.apache.hive" % "hive-metastore" % hiveVersion,
437+
"org.apache.hive" % "hive-exec" % hiveVersion,
438+
"org.apache.hive" % "hive-serde" % hiveVersion
439+
),
440+
// Multiple queries rely on the TestHive singleton. See comments there for more details.
441+
parallelExecution in Test := false,
442+
// Supporting all SerDes requires us to depend on deprecated APIs, so we turn off the warnings
443+
// only for this subproject.
444+
scalacOptions <<= scalacOptions map { currentOpts: Seq[String] =>
445+
currentOpts.filterNot(_ == "-deprecation")
446+
},
447+
initialCommands in console :=
448+
"""
449+
|import org.apache.spark.sql.catalyst.analysis._
450+
|import org.apache.spark.sql.catalyst.dsl._
451+
|import org.apache.spark.sql.catalyst.errors._
452+
|import org.apache.spark.sql.catalyst.expressions._
453+
|import org.apache.spark.sql.catalyst.plans.logical._
454+
|import org.apache.spark.sql.catalyst.rules._
455+
|import org.apache.spark.sql.catalyst.types._
456+
|import org.apache.spark.sql.catalyst.util._
457+
|import org.apache.spark.sql.execution
458+
|import org.apache.spark.sql.hive._
459+
|import org.apache.spark.sql.hive.TestHive._
460+
|import org.apache.spark.sql.parquet.ParquetTestData""".stripMargin
461+
)
462+
463+
def streamingSettings = sharedSettings ++ Seq(
464+
name := "spark-streaming",
465+
previousArtifact := sparkPreviousArtifact("spark-streaming")
466+
)
467+
414468
def yarnCommonSettings = sharedSettings ++ Seq(
415469
unmanagedSourceDirectories in Compile <++= baseDirectory { base =>
416470
Seq(
417-
base / "../common/src/main/scala"
471+
base / "../common/src/main/scala"
418472
)
419473
},
420474

421475
unmanagedSourceDirectories in Test <++= baseDirectory { base =>
422476
Seq(
423-
base / "../common/src/test/scala"
477+
base / "../common/src/test/scala"
424478
)
425479
}
426480

@@ -481,13 +535,15 @@ object SparkBuild extends Build {
481535

482536
def twitterSettings() = sharedSettings ++ Seq(
483537
name := "spark-streaming-twitter",
538+
previousArtifact := sparkPreviousArtifact("spark-streaming-twitter"),
484539
libraryDependencies ++= Seq(
485540
"org.twitter4j" % "twitter4j-stream" % "3.0.3" excludeAll(excludeNetty)
486541
)
487542
)
488543

489544
def kafkaSettings() = sharedSettings ++ Seq(
490545
name := "spark-streaming-kafka",
546+
previousArtifact := sparkPreviousArtifact("spark-streaming-kafka"),
491547
libraryDependencies ++= Seq(
492548
"com.github.sgroschupf" % "zkclient" % "0.1" excludeAll(excludeNetty),
493549
"org.apache.kafka" %% "kafka" % "0.8.0"
@@ -500,20 +556,23 @@ object SparkBuild extends Build {
500556

501557
def flumeSettings() = sharedSettings ++ Seq(
502558
name := "spark-streaming-flume",
559+
previousArtifact := sparkPreviousArtifact("spark-streaming-flume"),
503560
libraryDependencies ++= Seq(
504561
"org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty)
505562
)
506563
)
507564

508565
def zeromqSettings() = sharedSettings ++ Seq(
509566
name := "spark-streaming-zeromq",
567+
previousArtifact := sparkPreviousArtifact("spark-streaming-zeromq"),
510568
libraryDependencies ++= Seq(
511-
"org.spark-project.akka" %% "akka-zeromq" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty)
569+
"org.spark-project.akka" %% "akka-zeromq" % akkaVersion excludeAll(excludeNetty)
512570
)
513571
)
514572

515573
def mqttSettings() = streamingSettings ++ Seq(
516574
name := "spark-streaming-mqtt",
575+
previousArtifact := sparkPreviousArtifact("spark-streaming-mqtt"),
517576
libraryDependencies ++= Seq("org.eclipse.paho" % "mqtt-client" % "0.4.0")
518577
)
519578
}

0 commit comments

Comments
 (0)