Skip to content

Commit b8b3450

Browse files
committed
Update tests.
1 parent 2ac94f7 commit b8b3450

File tree

1 file changed

+60
-10
lines changed

1 file changed

+60
-10
lines changed

sql/hive/src/test/scala/org/apache/spark/sql/parquet/parquetSuites.scala

Lines changed: 60 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,9 @@ class ParquetDataSourceOnMetastoreSuite extends ParquetMetastoreSuiteBase {
122122
override def beforeAll(): Unit = {
123123
super.beforeAll()
124124

125+
val rdd = sparkContext.parallelize((1 to 10).map(i => s"""{"a":$i, "b":"str${i}"}"""))
126+
jsonRDD(rdd).registerTempTable("jt")
127+
125128
sql(
126129
"""
127130
|create table test_parquet
@@ -140,6 +143,7 @@ class ParquetDataSourceOnMetastoreSuite extends ParquetMetastoreSuiteBase {
140143

141144
override def afterAll(): Unit = {
142145
super.afterAll()
146+
sql("DROP TABLE IF EXISTS jt")
143147
sql("DROP TABLE IF EXISTS test_parquet")
144148

145149
setConf(SQLConf.PARQUET_USE_DATA_SOURCE_API, originalConf.toString)
@@ -153,14 +157,63 @@ class ParquetDataSourceOnMetastoreSuite extends ParquetMetastoreSuiteBase {
153157
checkAnswer(sql("SELECT count(INTFIELD) FROM TEST_parquet"), Row(0))
154158
}
155159

156-
test("scan a parquet table created through a CTAS statement") {
157-
val originalConvertMetastore = getConf("spark.sql.hive.convertMetastoreParquet", "true")
158-
val originalUseDataSource = getConf(SQLConf.PARQUET_USE_DATA_SOURCE_API, "true")
159-
setConf("spark.sql.hive.convertMetastoreParquet", "true")
160-
setConf(SQLConf.PARQUET_USE_DATA_SOURCE_API, "true")
160+
test("insert into an empty parquet table") {
161+
sql(
162+
"""
163+
|create table test_insert_parquet
164+
|(
165+
| intField INT,
166+
| stringField STRING
167+
|)
168+
|ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
169+
|STORED AS
170+
| INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
171+
| OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
172+
""".stripMargin)
161173

162-
val rdd = sparkContext.parallelize((1 to 10).map(i => s"""{"a":$i, "b":"str${i}"}"""))
163-
jsonRDD(rdd).registerTempTable("jt")
174+
// Insert into am empty table.
175+
sql("insert into table test_insert_parquet select a, b from jt where jt.a > 5")
176+
checkAnswer(
177+
sql(s"SELECT intField, stringField FROM test_insert_parquet WHERE intField < 8"),
178+
Row(6, "str6") :: Row(7, "str7") :: Nil
179+
)
180+
// Insert overwrite.
181+
sql("insert overwrite table test_insert_parquet select a, b from jt where jt.a < 5")
182+
checkAnswer(
183+
sql(s"SELECT intField, stringField FROM test_insert_parquet WHERE intField > 2"),
184+
Row(3, "str3") :: Row(4, "str4") :: Nil
185+
)
186+
sql("DROP TABLE IF EXISTS test_insert_parquet")
187+
188+
// Create it again.
189+
sql(
190+
"""
191+
|create table test_insert_parquet
192+
|(
193+
| intField INT,
194+
| stringField STRING
195+
|)
196+
|ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
197+
|STORED AS
198+
| INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
199+
| OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
200+
""".stripMargin)
201+
// Insert overwrite an empty table.
202+
sql("insert overwrite table test_insert_parquet select a, b from jt where jt.a < 5")
203+
checkAnswer(
204+
sql(s"SELECT intField, stringField FROM test_insert_parquet WHERE intField > 2"),
205+
Row(3, "str3") :: Row(4, "str4") :: Nil
206+
)
207+
// Insert into the table.
208+
sql("insert into table test_insert_parquet select a, b from jt")
209+
checkAnswer(
210+
sql(s"SELECT intField, stringField FROM test_insert_parquet"),
211+
(1 to 10).map(i => Row(i, s"str$i")) ++ (1 to 4).map(i => Row(i, s"str$i"))
212+
)
213+
sql("DROP TABLE IF EXISTS test_insert_parquet")
214+
}
215+
216+
test("scan a parquet table created through a CTAS statement") {
164217
sql(
165218
"""
166219
|create table test_parquet_ctas ROW FORMAT
@@ -183,10 +236,7 @@ class ParquetDataSourceOnMetastoreSuite extends ParquetMetastoreSuiteBase {
183236
s"test_parquet_ctas should be converted to ${classOf[ParquetRelation2].getCanonicalName}")
184237
}
185238

186-
sql("DROP TABLE IF EXISTS jt")
187239
sql("DROP TABLE IF EXISTS test_parquet_ctas")
188-
setConf("spark.sql.hive.convertMetastoreParquet", originalConvertMetastore)
189-
setConf(SQLConf.PARQUET_USE_DATA_SOURCE_API, originalUseDataSource)
190240
}
191241
}
192242

0 commit comments

Comments
 (0)