Skip to content

Commit 6b80ce4

Browse files
dongjoon-hyungatorsmile
authored andcommitted
[SPARK-19809][SQL][TEST][FOLLOWUP] Move the test case to HiveOrcQuerySuite
## What changes were proposed in this pull request? As a follow-up of #19948 , this PR moves the test case and adds comments. ## How was this patch tested? Pass the Jenkins. Author: Dongjoon Hyun <[email protected]> Closes #19960 from dongjoon-hyun/SPARK-19809-2.
1 parent 0e36ba6 commit 6b80ce4

File tree

2 files changed

+47
-37
lines changed

2 files changed

+47
-37
lines changed

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala

Lines changed: 0 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2153,40 +2153,4 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
21532153
}
21542154
}
21552155
}
2156-
2157-
test("SPARK-22267 Spark SQL incorrectly reads ORC files when column order is different") {
2158-
Seq("native", "hive").foreach { orcImpl =>
2159-
withSQLConf(SQLConf.ORC_IMPLEMENTATION.key -> orcImpl) {
2160-
withTempPath { f =>
2161-
val path = f.getCanonicalPath
2162-
Seq(1 -> 2).toDF("c1", "c2").write.orc(path)
2163-
checkAnswer(spark.read.orc(path), Row(1, 2))
2164-
2165-
withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "true") { // default since 2.3.0
2166-
withTable("t") {
2167-
sql(s"CREATE EXTERNAL TABLE t(c2 INT, c1 INT) STORED AS ORC LOCATION '$path'")
2168-
checkAnswer(spark.table("t"), Row(2, 1))
2169-
}
2170-
}
2171-
}
2172-
}
2173-
}
2174-
}
2175-
2176-
test("SPARK-19809 NullPointerException on zero-size ORC file") {
2177-
Seq("native", "hive").foreach { orcImpl =>
2178-
withSQLConf(SQLConf.ORC_IMPLEMENTATION.key -> orcImpl) {
2179-
withTempPath { dir =>
2180-
withTable("spark_19809") {
2181-
sql(s"CREATE TABLE spark_19809(a int) STORED AS ORC LOCATION '$dir'")
2182-
Files.touch(new File(s"${dir.getCanonicalPath}", "zero.orc"))
2183-
2184-
withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "true") { // default since 2.3.0
2185-
checkAnswer(sql("SELECT * FROM spark_19809"), Seq.empty)
2186-
}
2187-
}
2188-
}
2189-
}
2190-
}
2191-
}
21922156
}

sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,11 @@
1717

1818
package org.apache.spark.sql.hive.orc
1919

20-
import org.apache.spark.sql.AnalysisException
20+
import java.io.File
21+
22+
import com.google.common.io.Files
23+
24+
import org.apache.spark.sql.{AnalysisException, Row}
2125
import org.apache.spark.sql.catalyst.catalog.HiveTableRelation
2226
import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
2327
import org.apache.spark.sql.execution.datasources.orc.OrcQueryTest
@@ -162,4 +166,46 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
162166
}
163167
}
164168
}
169+
170+
// Since Hive 1.2.1 library code path still has this problem, users may hit this
171+
// when spark.sql.hive.convertMetastoreOrc=false. However, after SPARK-22279,
172+
// Apache Spark with the default configuration doesn't hit this bug.
173+
test("SPARK-22267 Spark SQL incorrectly reads ORC files when column order is different") {
174+
Seq("native", "hive").foreach { orcImpl =>
175+
withSQLConf(SQLConf.ORC_IMPLEMENTATION.key -> orcImpl) {
176+
withTempPath { f =>
177+
val path = f.getCanonicalPath
178+
Seq(1 -> 2).toDF("c1", "c2").write.orc(path)
179+
checkAnswer(spark.read.orc(path), Row(1, 2))
180+
181+
withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "true") { // default since 2.3.0
182+
withTable("t") {
183+
sql(s"CREATE EXTERNAL TABLE t(c2 INT, c1 INT) STORED AS ORC LOCATION '$path'")
184+
checkAnswer(spark.table("t"), Row(2, 1))
185+
}
186+
}
187+
}
188+
}
189+
}
190+
}
191+
192+
// Since Hive 1.2.1 library code path still has this problem, users may hit this
193+
// when spark.sql.hive.convertMetastoreOrc=false. However, after SPARK-22279,
194+
// Apache Spark with the default configuration doesn't hit this bug.
195+
test("SPARK-19809 NullPointerException on zero-size ORC file") {
196+
Seq("native", "hive").foreach { orcImpl =>
197+
withSQLConf(SQLConf.ORC_IMPLEMENTATION.key -> orcImpl) {
198+
withTempPath { dir =>
199+
withTable("spark_19809") {
200+
sql(s"CREATE TABLE spark_19809(a int) STORED AS ORC LOCATION '$dir'")
201+
Files.touch(new File(s"${dir.getCanonicalPath}", "zero.orc"))
202+
203+
withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> "true") { // default since 2.3.0
204+
checkAnswer(spark.table("spark_19809"), Seq.empty)
205+
}
206+
}
207+
}
208+
}
209+
}
210+
}
165211
}

0 commit comments

Comments
 (0)