Skip to content

Commit 734bac9

Browse files
committed
Explicitly set the implicit parameters
1 parent ca03559 commit 734bac9

File tree

1 file changed

+14
-7
lines changed
  • core/src/main/scala/org/apache/spark/rdd

1 file changed

+14
-7
lines changed

core/src/main/scala/org/apache/spark/rdd/RDD.scala

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1176,13 +1176,17 @@ abstract class RDD[T: ClassTag](
11761176
def saveAsTextFile(path: String) {
11771177
// https://issues.apache.org/jira/browse/SPARK-2075
11781178
// NullWritable is a Comparable rather than Comparable[NullWritable] in Hadoop 1.+,
1179-
// so the compiler cannot find an implicit Ordering for it. It will generate different
1180-
// anonymous classes for `saveAsTextFile` in Hadoop 1.+ and Hadoop 2.+. Therefore, here we
1181-
// provide an Ordering for NullWritable so that the compiler will generate same codes.
1182-
implicit val nullWritableOrdering = new Ordering[NullWritable] {
1179+
// so the compiler cannot find an implicit Ordering for it and will use the default `null`.
1180+
// It will generate different anonymous classes for `saveAsTextFile` in Hadoop 1.+ and
1181+
// Hadoop 2.+. Therefore, here we provide an Ordering for NullWritable so that the compiler
1182+
// will generate same bytecode.
1183+
val nullWritableOrdering = new Ordering[NullWritable] {
11831184
override def compare(x: NullWritable, y: NullWritable): Int = 0
11841185
}
1185-
this.map(x => (NullWritable.get(), new Text(x.toString)))
1186+
val nullWritableClassTag = implicitly[ClassTag[NullWritable]]
1187+
val textClassTag = implicitly[ClassTag[Text]]
1188+
val r = this.map(x => (NullWritable.get(), new Text(x.toString)))
1189+
RDD.rddToPairRDDFunctions(r)(nullWritableClassTag, textClassTag, nullWritableOrdering)
11861190
.saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path)
11871191
}
11881192

@@ -1191,10 +1195,13 @@ abstract class RDD[T: ClassTag](
11911195
*/
11921196
def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]) {
11931197
// https://issues.apache.org/jira/browse/SPARK-2075
1194-
implicit val nullWritableOrdering = new Ordering[NullWritable] {
1198+
val nullWritableOrdering = new Ordering[NullWritable] {
11951199
override def compare(x: NullWritable, y: NullWritable): Int = 0
11961200
}
1197-
this.map(x => (NullWritable.get(), new Text(x.toString)))
1201+
val nullWritableClassTag = implicitly[ClassTag[NullWritable]]
1202+
val textClassTag = implicitly[ClassTag[Text]]
1203+
val r = this.map(x => (NullWritable.get(), new Text(x.toString)))
1204+
RDD.rddToPairRDDFunctions(r)(nullWritableClassTag, textClassTag, nullWritableOrdering)
11981205
.saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path, codec)
11991206
}
12001207

0 commit comments

Comments
 (0)