From 5bf56a7152c2ae75d2e554c4f47a6946f8bf2ab0 Mon Sep 17 00:00:00 2001 From: tmalaska Date: Wed, 23 Apr 2014 09:40:43 -0400 Subject: [PATCH 1/3] Upgrade flume version --- external/flume/pom.xml | 2 +- project/SparkBuild.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/external/flume/pom.xml b/external/flume/pom.xml index f21963531574..5ec6dbeb32d5 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -61,7 +61,7 @@ org.apache.flume flume-ng-sdk - 1.2.0 + 1.4.0 org.jboss.netty diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 852358501af5..6e0542cbbd20 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -605,7 +605,7 @@ object SparkBuild extends Build { name := "spark-streaming-flume", previousArtifact := sparkPreviousArtifact("spark-streaming-flume"), libraryDependencies ++= Seq( - "org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty) + "org.apache.flume" % "flume-ng-sdk" % "1.4.0" % "compile" excludeAll(excludeNetty) ) ) From 159c3f197f371336ac5e2c4f6c81926eed0aa85a Mon Sep 17 00:00:00 2001 From: tmalaska Date: Thu, 24 Apr 2014 10:05:57 -0400 Subject: [PATCH 2/3] fixed the flume pom file issues --- project/SparkBuild.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 6e0542cbbd20..916aec3bf58d 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -313,6 +313,7 @@ object SparkBuild extends Build { val excludeCurator = ExclusionRule(organization = "org.apache.curator") val excludePowermock = ExclusionRule(organization = "org.powermock") val excludeFastutil = ExclusionRule(organization = "it.unimi.dsi") + val excludeThrift = ExclusionRule(organization = "org.apache.thrift") def sparkPreviousArtifact(id: String, organization: String = "org.apache.spark", version: String = "0.9.0-incubating", crossVersion: String = "2.10"): Option[sbt.ModuleID] = { @@ -605,7 +606,8 @@ object SparkBuild extends Build { name := "spark-streaming-flume", previousArtifact := sparkPreviousArtifact("spark-streaming-flume"), libraryDependencies ++= Seq( - "org.apache.flume" % "flume-ng-sdk" % "1.4.0" % "compile" excludeAll(excludeNetty) + "org.apache.flume" % "flume-ng-sdk" % "1.4.0" % "compile" excludeAll(excludeNetty, excludeThrift), + "org.apache.thrift" % "libthrift" % "0.8.0" % "compile" ) ) From 79492c8f2f08d706b6ae97bec22325bd9b87ef4f Mon Sep 17 00:00:00 2001 From: tmalaska Date: Thu, 24 Apr 2014 18:24:18 -0400 Subject: [PATCH 3/3] excluded all thrift --- external/flume/pom.xml | 4 ++++ project/SparkBuild.scala | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 5ec6dbeb32d5..b84ca0c5e1dd 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -67,6 +67,10 @@ org.jboss.netty netty + + org.apache.thrift + libthrift + diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 916aec3bf58d..993486b927d6 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -606,8 +606,7 @@ object SparkBuild extends Build { name := "spark-streaming-flume", previousArtifact := sparkPreviousArtifact("spark-streaming-flume"), libraryDependencies ++= Seq( - "org.apache.flume" % "flume-ng-sdk" % "1.4.0" % "compile" excludeAll(excludeNetty, excludeThrift), - "org.apache.thrift" % "libthrift" % "0.8.0" % "compile" + "org.apache.flume" % "flume-ng-sdk" % "1.4.0" % "compile" excludeAll(excludeNetty, excludeThrift) ) )