From 964e3f7df2515c2d47ce301be4784fb961483bb3 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 18 Mar 2014 14:43:22 -0500 Subject: [PATCH 1/2] SPARK-1203 fix saving to hdfs from yarn --- core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index b0d322fe27bd5..a1594e0dd0437 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -711,6 +711,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) if (valueClass == null) { throw new SparkException("Output value class not set") } + SparkHadoopUtil.get.addCredentials(conf) logDebug("Saving as hadoop file of type (" + keyClass.getSimpleName + ", " + valueClass.getSimpleName + ")") From 4fd5ded64ce2125f7f756bca27f9f194c78a6346 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 18 Mar 2014 15:10:46 -0500 Subject: [PATCH 2/2] adding import --- core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index a1594e0dd0437..75afacf4013f8 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.lib.output.{FileOutputFormat => NewFileOutput import org.apache.hadoop.mapred.SparkHadoopWriter import org.apache.spark._ +import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.Partitioner.defaultPartitioner import org.apache.spark.SparkContext._ import org.apache.spark.partial.{BoundedDouble, PartialResult}