Skip to content

Commit e0490e2

Browse files
guowei2marmbrus
authored andcommitted
[SPARK-5118][SQL] Fix: create table test stored as parquet as select ..
Author: guowei2 <[email protected]> Closes #3921 from guowei2/SPARK-5118 and squashes the following commits: b1ba3be [guowei2] add table file check in test case 9da56f8 [guowei2] test case only run in Shim13 112a0b6 [guowei2] add test case 187c7d8 [guowei2] Fix: create table test stored as parquet as select ..
1 parent 548c9c2 commit e0490e2

File tree

2 files changed

+29
-1
lines changed

2 files changed

+29
-1
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
554554
"TOK_TBLTEXTFILE", // Stored as TextFile
555555
"TOK_TBLRCFILE", // Stored as RCFile
556556
"TOK_TBLORCFILE", // Stored as ORC File
557+
"TOK_TBLPARQUETFILE", // Stored as PARQUET
557558
"TOK_TABLEFILEFORMAT", // User-provided InputFormat and OutputFormat
558559
"TOK_STORAGEHANDLER", // Storage handler
559560
"TOK_TABLELOCATION",

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import org.apache.spark.sql.QueryTest
2222
import org.apache.spark.sql.Row
2323
import org.apache.spark.sql.hive.test.TestHive._
2424
import org.apache.spark.sql.types._
25+
import org.apache.spark.util.Utils
26+
import org.apache.spark.sql.hive.HiveShim
2527

2628
case class Nested1(f1: Nested2)
2729
case class Nested2(f2: Nested3)
@@ -102,6 +104,31 @@ class SQLQuerySuite extends QueryTest {
102104
"org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe",
103105
"serde_p1=p1", "serde_p2=p2", "tbl_p1=p11", "tbl_p2=p22","MANAGED_TABLE"
104106
)
107+
108+
if (HiveShim.version =="0.13.1") {
109+
sql(
110+
"""CREATE TABLE ctas5
111+
| STORED AS parquet AS
112+
| SELECT key, value
113+
| FROM src
114+
| ORDER BY key, value""".stripMargin).collect
115+
116+
checkExistence(sql("DESC EXTENDED ctas5"), true,
117+
"name:key", "type:string", "name:value", "ctas5",
118+
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
119+
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
120+
"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
121+
"MANAGED_TABLE"
122+
)
123+
124+
val default = getConf("spark.sql.hive.convertMetastoreParquet", "true")
125+
// use the Hive SerDe for parquet tables
126+
sql("set spark.sql.hive.convertMetastoreParquet = false")
127+
checkAnswer(
128+
sql("SELECT key, value FROM ctas5 ORDER BY key, value"),
129+
sql("SELECT key, value FROM src ORDER BY key, value").collect().toSeq)
130+
sql(s"set spark.sql.hive.convertMetastoreParquet = $default")
131+
}
105132
}
106133

107134
test("command substitution") {
@@ -159,7 +186,7 @@ class SQLQuerySuite extends QueryTest {
159186
test("test CTAS") {
160187
checkAnswer(sql("CREATE TABLE test_ctas_123 AS SELECT key, value FROM src"), Seq.empty[Row])
161188
checkAnswer(
162-
sql("SELECT key, value FROM test_ctas_123 ORDER BY key"),
189+
sql("SELECT key, value FROM test_ctas_123 ORDER BY key"),
163190
sql("SELECT key, value FROM src ORDER BY key").collect().toSeq)
164191
}
165192

0 commit comments

Comments
 (0)