diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala index ce851c58cc4f..39a876316053 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala @@ -228,13 +228,4 @@ class OrcFileFormat case _ => false } - - override def supportFieldName(name: String): Boolean = { - try { - TypeDescription.fromString(s"struct<`$name`:int>") - true - } catch { - case _: IllegalArgumentException => false - } - } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 523a8e242e7e..ffc3db31c90d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -4262,6 +4262,25 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark Row(2, 4, 6, 8, 10, 12, 14, 16, 18, 20) :: Nil) } } + + test("SPARK-37965: Spark support read/write orc file with invalid char in field name") { + withTempDir { dir => + Seq((1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), (2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22)) + .toDF("max(t)", "max(t", "=", "\n", ";", "a b", "{", ".", "a.b", "a", ",") + .repartition(1) + .write.mode(SaveMode.Overwrite).orc(dir.getAbsolutePath) + val df = spark.read.orc(dir.getAbsolutePath) + checkAnswer(df, + Row(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) :: + Row(2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22) :: Nil) + assert(df.schema.names.sameElements( + Array("max(t)", "max(t", "=", "\n", ";", "a b", "{", ".", "a.b", "a", ","))) + checkAnswer(df.select("`max(t)`", "`a b`", "`{`", "`.`", "`a.b`"), + Row(1, 6, 7, 8, 9) :: Row(2, 12, 14, 16, 18) :: Nil) + checkAnswer(df.where("`a.b` > 10"), + Row(2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22) :: Nil) + } + } } case class Foo(bar: Option[String]) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index e690d026053d..d3f5d7613ace 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -2213,8 +2213,9 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi } test("SPARK-32889: ORC table column name supports special characters") { - // " " "," is not allowed. - Seq("$", ";", "{", "}", "(", ")", "\n", "\t", "=").foreach { name => + // "," is not allowed since cannot create a table having a column whose name + // contains commas in Hive metastore. + Seq("$", ";", "{", "}", "(", ")", "\n", "\t", "=", " ", "a b").foreach { name => val source = "ORC" Seq(s"CREATE TABLE t32889(`$name` INT) USING $source", s"CREATE TABLE t32889 STORED AS $source AS SELECT 1 `$name`",