[SPARK-32524][SQL][TESTS] SharedSparkSession should clean up InMemoryRelation.ser

Author: dongjoon-hyun

sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala

@@ -277,6 +277,9 @@ object InMemoryRelation {
     ser.get
   }
 
+  /* Visible for testing */
+  private[spark] def clearSerializer(): Unit = synchronized { ser = None }
+
   def convertToColumnarIfPossible(plan: SparkPlan): SparkPlan = plan match {
     case gen: WholeStageCodegenExec => gen.child match {
       case c2r: ColumnarToRowTransition => c2r.child match {

sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala

@@ -26,6 +26,7 @@ import org.apache.spark.{DebugFilesystem, SparkConf}
 import org.apache.spark.internal.config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK
 import org.apache.spark.sql.{SparkSession, SQLContext}
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
+import org.apache.spark.sql.execution.columnar.InMemoryRelation.clearSerializer
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 
 trait SharedSparkSession extends SQLTestUtils with SharedSparkSessionBase {
@@ -63,6 +64,7 @@ trait SharedSparkSessionBase
   with Eventually { self: Suite =>
 
   protected def sparkConf = {
+    clearSerializer()
     val conf = new SparkConf()
       .set("spark.hadoop.fs.file.impl", classOf[DebugFilesystem].getName)
       .set(UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true)