Skip to content

Commit e3c0fbe

Browse files
committed
[SPARK-42071][CORE] Register scala.math.Ordering$Reverse to KyroSerializer
### What changes were proposed in this pull request? This PR aims to register `scala.math.Ordering$Reverse` to KyroSerializer. ### Why are the changes needed? Scala 2.12.12 added a new class 'Reverse' via scala/scala#8965. This affects Apache Spark 3.2.0+. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs with newly added test case. Closes apache#39578 from dongjoon-hyun/SPARK-42071. Authored-by: Dongjoon Hyun <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent d9996c6 commit e3c0fbe

File tree

2 files changed

+14
-1
lines changed

2 files changed

+14
-1
lines changed

core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ class KryoSerializer(conf: SparkConf)
219219
kryo.register(Nil.getClass)
220220
kryo.register(Utils.classForName("scala.collection.immutable.$colon$colon"))
221221
kryo.register(Utils.classForName("scala.collection.immutable.Map$EmptyMap$"))
222+
kryo.register(Utils.classForName("scala.math.Ordering$Reverse"))
222223
kryo.register(classOf[ArrayBuffer[Any]])
223224
kryo.register(classOf[Array[Array[Byte]]])
224225

core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage
3838
import org.apache.spark.scheduler.HighlyCompressedMapStatus
3939
import org.apache.spark.serializer.KryoTest._
4040
import org.apache.spark.storage.BlockManagerId
41-
import org.apache.spark.util.ThreadUtils
41+
import org.apache.spark.util.{ThreadUtils, Utils}
4242
import org.apache.spark.util.collection.OpenHashMap
4343

4444
class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
@@ -174,6 +174,18 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
174174
mutable.HashMap(1 -> "one", 2 -> "two", 3 -> "three")))
175175
}
176176

177+
test("SPARK-42071: Register scala.math.Ordering$Reverse") {
178+
val conf = new SparkConf(false)
179+
conf.set(KRYO_REGISTRATION_REQUIRED, true)
180+
181+
val ser = new KryoSerializer(conf).newInstance()
182+
def check[T: ClassTag](t: T): Unit = {
183+
assert(ser.deserialize[T](ser.serialize(t)) === t)
184+
}
185+
// Scala 2.12.12 added a new class 'Reverse' via https://github.com/scala/scala/pull/8965
186+
check(Utils.classForName("scala.math.Ordering$Reverse"))
187+
}
188+
177189
test("Bug: SPARK-10251") {
178190
val ser = new KryoSerializer(conf.clone.set(KRYO_REGISTRATION_REQUIRED, true))
179191
.newInstance()

0 commit comments

Comments
 (0)