diff --git a/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScan.scala b/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScan.scala index d0f38c12427c3..763b9abe4f91b 100644 --- a/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScan.scala +++ b/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroScan.scala @@ -70,10 +70,6 @@ case class AvroScan( override def hashCode(): Int = super.hashCode() - override def description(): String = { - super.description() + ", PushedFilters: " + pushedFilters.mkString("[", ", ", "]") - } - override def getMetaData(): Map[String, String] = { super.getMetaData() ++ Map("PushedFilters" -> seqToString(pushedFilters)) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScan.scala index d81223b48a53f..734f8165aff3d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScan.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVScan.scala @@ -91,10 +91,6 @@ case class CSVScan( override def hashCode(): Int = super.hashCode() - override def description(): String = { - super.description() + ", PushedFilters: " + pushedFilters.mkString("[", ", ", "]") - } - override def getMetaData(): Map[String, String] = { super.getMetaData() ++ Map("PushedFilters" -> seqToString(pushedFilters)) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScan.scala index 9ab367136fc97..c9a3a6f5e7f2d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScan.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonScan.scala @@ -91,7 +91,7 @@ case class JsonScan( override def hashCode(): Int = super.hashCode() - override def description(): String = { - super.description() + ", PushedFilters: " + pushedFilters.mkString("[", ", ", "]") + override def getMetaData(): Map[String, String] = { + super.getMetaData() ++ Map("PushedFilters" -> pushedFilters.mkString("[", ", ", "]")) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala index ccb9ca9c6b3f4..072ab26774e52 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala @@ -92,12 +92,6 @@ case class OrcScan( ("[]", "[]") } - override def description(): String = { - super.description() + ", PushedFilters: " + seqToString(pushedFilters) + - ", PushedAggregation: " + pushedAggregationsStr + - ", PushedGroupBy: " + pushedGroupByStr - } - override def getMetaData(): Map[String, String] = { super.getMetaData() ++ Map("PushedFilters" -> seqToString(pushedFilters)) ++ Map("PushedAggregation" -> pushedAggregationsStr) ++ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScan.scala index ff0b38880fd8e..619a8fe66e3b4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScan.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScan.scala @@ -127,12 +127,6 @@ case class ParquetScan( ("[]", "[]") } - override def description(): String = { - super.description() + ", PushedFilters: " + seqToString(pushedFilters) + - ", PushedAggregation: " + pushedAggregationsStr + - ", PushedGroupBy: " + pushedGroupByStr - } - override def getMetaData(): Map[String, String] = { super.getMetaData() ++ Map("PushedFilters" -> seqToString(pushedFilters)) ++ Map("PushedAggregation" -> pushedAggregationsStr) ++ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala index 50c3b8fbf4857..b5353455dc275 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala @@ -462,17 +462,8 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite withTempDir { dir => Seq("parquet", "orc", "csv", "json").foreach { fmt => val basePath = dir.getCanonicalPath + "/" + fmt - val pushFilterMaps = Map ( - "parquet" -> - "|PushedFilters: \\[IsNotNull\\(value\\), GreaterThan\\(value,2\\)\\]", - "orc" -> - "|PushedFilters: \\[IsNotNull\\(value\\), GreaterThan\\(value,2\\)\\]", - "csv" -> - "|PushedFilters: \\[IsNotNull\\(value\\), GreaterThan\\(value,2\\)\\]", - "json" -> - "|remove_marker" - ) - val expected_plan_fragment1 = + + val expectedPlanFragment = s""" |\\(1\\) BatchScan $fmt file:$basePath |Output \\[2\\]: \\[value#x, id#x\\] @@ -480,9 +471,9 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite |Format: $fmt |Location: InMemoryFileIndex\\([0-9]+ paths\\)\\[.*\\] |PartitionFilters: \\[isnotnull\\(id#x\\), \\(id#x > 1\\)\\] - ${pushFilterMaps.get(fmt).get} + |PushedFilters: \\[IsNotNull\\(value\\), GreaterThan\\(value,2\\)\\] |ReadSchema: struct\\ - |""".stripMargin.replaceAll("\nremove_marker", "").trim + |""".stripMargin.trim spark.range(10) .select(col("id"), col("id").as("value")) @@ -500,7 +491,7 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite .format(fmt) .load(basePath).where($"id" > 1 && $"value" > 2) val normalizedOutput = getNormalizedExplain(df, FormattedMode) - assert(expected_plan_fragment1.r.findAllMatchIn(normalizedOutput).length == 1) + assert(expectedPlanFragment.r.findAllMatchIn(normalizedOutput).length == 1) } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceAggregatePushDownSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceAggregatePushDownSuite.scala index a68d9b951b748..e8fae210fa4bd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceAggregatePushDownSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceAggregatePushDownSuite.scala @@ -347,7 +347,8 @@ trait FileSourceAggregatePushDownSuite spark.read.format(format).load(file.getCanonicalPath).createOrReplaceTempView("test") Seq("false", "true").foreach { enableVectorizedReader => withSQLConf(aggPushDownEnabledKey -> "true", - vectorizedReaderEnabledKey -> enableVectorizedReader) { + vectorizedReaderEnabledKey -> enableVectorizedReader, + SQLConf.MAX_METADATA_STRING_LENGTH.key -> "1000") { val testMinWithAllTypes = sql("SELECT min(StringCol), min(BooleanCol), min(ByteCol), " + "min(BinaryCol), min(ShortCol), min(IntegerCol), min(LongCol), min(FloatCol), " +