From 398dbabd08ca46c0a5acfb035f25b637ecff63cd Mon Sep 17 00:00:00 2001 From: Nan Zhu Date: Mon, 21 Apr 2014 15:54:43 -0400 Subject: [PATCH 1/3] SPARK-1556: bump jet3st version to 0.9.0 --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 09b527c76a5ae..f6cf5aca4bdd1 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -334,7 +334,7 @@ object SparkBuild extends Build { "colt" % "colt" % "1.2.0", "org.apache.mesos" % "mesos" % "0.13.0", "commons-net" % "commons-net" % "2.2", - "net.java.dev.jets3t" % "jets3t" % "0.7.1" excludeAll(excludeCommonsLogging), + "net.java.dev.jets3t" % "jets3t" % "0.9.0" excludeAll(excludeCommonsLogging), "org.apache.derby" % "derby" % "10.4.2.0" % "test", "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm), "org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty), From 775b19c5d90a0147b0d9c7cbcff65cc911cd961f Mon Sep 17 00:00:00 2001 From: Nan Zhu Date: Mon, 21 Apr 2014 16:13:28 -0400 Subject: [PATCH 2/3] append mvn modification --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cd204376de5db..f0654dcfd624d 100644 --- a/pom.xml +++ b/pom.xml @@ -514,7 +514,7 @@ net.java.dev.jets3t jets3t - 0.7.1 + 0.9.0 org.apache.hadoop From 3d6e824e5772073ce0a4e8379533460b845c1730 Mon Sep 17 00:00:00 2001 From: CodingCat Date: Mon, 28 Apr 2014 23:54:28 -0400 Subject: [PATCH 3/3] tell user to manually update the build file --- docs/index.md | 4 ++++ pom.xml | 2 +- project/SparkBuild.scala | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index 89ec5b05488a9..f68caa2aac515 100644 --- a/docs/index.md +++ b/docs/index.md @@ -67,6 +67,10 @@ In addition, if you wish to run Spark on [YARN](running-on-yarn.html), set Note that on Windows, you need to set the environment variables on separate lines, e.g., `set SPARK_HADOOP_VERSION=1.2.1`. +If you want to access Amazon S3 with a Spark compiled with Hadoop 2.3.0 or newer version, you have to manually update the dependency of +jets3t to 0.9.0 in the Spark build file (pom.xml or project/SparkBuild.scala), as jets3t 0.9.0 is introduced in the latest +versions of Hadoop and not backward compatible. + # Where to Go from Here **Programming guides:** diff --git a/pom.xml b/pom.xml index f0654dcfd624d..cd204376de5db 100644 --- a/pom.xml +++ b/pom.xml @@ -514,7 +514,7 @@ net.java.dev.jets3t jets3t - 0.9.0 + 0.7.1 org.apache.hadoop diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f6cf5aca4bdd1..09b527c76a5ae 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -334,7 +334,7 @@ object SparkBuild extends Build { "colt" % "colt" % "1.2.0", "org.apache.mesos" % "mesos" % "0.13.0", "commons-net" % "commons-net" % "2.2", - "net.java.dev.jets3t" % "jets3t" % "0.9.0" excludeAll(excludeCommonsLogging), + "net.java.dev.jets3t" % "jets3t" % "0.7.1" excludeAll(excludeCommonsLogging), "org.apache.derby" % "derby" % "10.4.2.0" % "test", "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm), "org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty),