diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala index 1234a8082c09..c9a9657f8764 100644 --- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala +++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.serializer -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, FileInputStream, FileOutputStream} +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} import java.nio.ByteBuffer import scala.collection.JavaConverters._ @@ -32,7 +32,6 @@ import org.apache.spark.{SharedSparkContext, SparkConf, SparkFunSuite} import org.apache.spark.scheduler.HighlyCompressedMapStatus import org.apache.spark.serializer.KryoTest._ import org.apache.spark.storage.BlockManagerId -import org.apache.spark.util.Utils class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext { conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") @@ -365,30 +364,6 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext { assert(thrown.getCause.isInstanceOf[KryoException]) } - test("SPARK-12222: deserialize RoaringBitmap throw Buffer underflow exception") { - val dir = Utils.createTempDir() - val tmpfile = dir.toString + "/RoaringBitmap" - val outStream = new FileOutputStream(tmpfile) - val output = new KryoOutput(outStream) - val bitmap = new RoaringBitmap - bitmap.add(1) - bitmap.add(3) - bitmap.add(5) - // Ignore Kryo because it doesn't use writeObject - bitmap.serialize(new KryoOutputObjectOutputBridge(null, output)) - output.flush() - output.close() - - val inStream = new FileInputStream(tmpfile) - val input = new KryoInput(inStream) - val ret = new RoaringBitmap - // Ignore Kryo because it doesn't use readObject - ret.deserialize(new KryoInputObjectInputBridge(null, input)) - input.close() - assert(ret == bitmap) - Utils.deleteRecursively(dir) - } - test("KryoOutputObjectOutputBridge.writeObject and KryoInputObjectInputBridge.readObject") { val kryo = new KryoSerializer(conf).newKryo() @@ -442,6 +417,14 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext { testSerializerInstanceReuse(autoReset = autoReset, referenceTracking = referenceTracking) } } + + test("SPARK-27216: test RoaringBitmap ser/dser with Kryo") { + val expected = new RoaringBitmap() + expected.add(1787) + val ser = new KryoSerializer(conf).newInstance() + val actual: RoaringBitmap = ser.deserialize(ser.serialize(expected)) + assert(actual === expected) + } } class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with SharedSparkContext { diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 4f0794d6f1a1..c9f219d6101f 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -1,5 +1,5 @@ JavaEWAH-0.3.2.jar -RoaringBitmap-0.5.11.jar +RoaringBitmap-0.7.45.jar ST4-4.0.4.jar activation-1.1.1.jar aircompressor-0.8.jar @@ -178,6 +178,7 @@ scala-reflect-2.11.8.jar scala-xml_2.11-1.0.5.jar scalap-2.11.8.jar shapeless_2.11-2.3.2.jar +shims-0.7.45.jar slf4j-api-1.7.16.jar slf4j-log4j12-1.7.16.jar snakeyaml-1.15.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index df2be777ff5a..8e99ea328c3d 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -1,5 +1,5 @@ JavaEWAH-0.3.2.jar -RoaringBitmap-0.5.11.jar +RoaringBitmap-0.7.45.jar ST4-4.0.4.jar activation-1.1.1.jar aircompressor-0.8.jar @@ -179,6 +179,7 @@ scala-reflect-2.11.8.jar scala-xml_2.11-1.0.5.jar scalap-2.11.8.jar shapeless_2.11-2.3.2.jar +shims-0.7.45.jar slf4j-api-1.7.16.jar slf4j-log4j12-1.7.16.jar snakeyaml-1.15.jar diff --git a/pom.xml b/pom.xml index cee4d1442c7d..9cb1a550de9a 100644 --- a/pom.xml +++ b/pom.xml @@ -573,7 +573,7 @@ org.roaringbitmap RoaringBitmap - 0.5.11 + 0.7.45 commons-net