@@ -122,6 +122,9 @@ class ParquetDataSourceOnMetastoreSuite extends ParquetMetastoreSuiteBase {
122122 override def beforeAll (): Unit = {
123123 super .beforeAll()
124124
125+ val rdd = sparkContext.parallelize((1 to 10 ).map(i => s """ {"a": $i, "b":"str ${i}"} """ ))
126+ jsonRDD(rdd).registerTempTable(" jt" )
127+
125128 sql(
126129 """
127130 |create table test_parquet
@@ -140,6 +143,7 @@ class ParquetDataSourceOnMetastoreSuite extends ParquetMetastoreSuiteBase {
140143
141144 override def afterAll (): Unit = {
142145 super .afterAll()
146+ sql(" DROP TABLE IF EXISTS jt" )
143147 sql(" DROP TABLE IF EXISTS test_parquet" )
144148
145149 setConf(SQLConf .PARQUET_USE_DATA_SOURCE_API , originalConf.toString)
@@ -153,14 +157,63 @@ class ParquetDataSourceOnMetastoreSuite extends ParquetMetastoreSuiteBase {
153157 checkAnswer(sql(" SELECT count(INTFIELD) FROM TEST_parquet" ), Row (0 ))
154158 }
155159
156- test(" scan a parquet table created through a CTAS statement" ) {
157- val originalConvertMetastore = getConf(" spark.sql.hive.convertMetastoreParquet" , " true" )
158- val originalUseDataSource = getConf(SQLConf .PARQUET_USE_DATA_SOURCE_API , " true" )
159- setConf(" spark.sql.hive.convertMetastoreParquet" , " true" )
160- setConf(SQLConf .PARQUET_USE_DATA_SOURCE_API , " true" )
160+ test(" insert into an empty parquet table" ) {
161+ sql(
162+ """
163+ |create table test_insert_parquet
164+ |(
165+ | intField INT,
166+ | stringField STRING
167+ |)
168+ |ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
169+ |STORED AS
170+ | INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
171+ | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
172+ """ .stripMargin)
161173
162- val rdd = sparkContext.parallelize((1 to 10 ).map(i => s """ {"a": $i, "b":"str ${i}"} """ ))
163- jsonRDD(rdd).registerTempTable(" jt" )
174+ // Insert into am empty table.
175+ sql(" insert into table test_insert_parquet select a, b from jt where jt.a > 5" )
176+ checkAnswer(
177+ sql(s " SELECT intField, stringField FROM test_insert_parquet WHERE intField < 8 " ),
178+ Row (6 , " str6" ) :: Row (7 , " str7" ) :: Nil
179+ )
180+ // Insert overwrite.
181+ sql(" insert overwrite table test_insert_parquet select a, b from jt where jt.a < 5" )
182+ checkAnswer(
183+ sql(s " SELECT intField, stringField FROM test_insert_parquet WHERE intField > 2 " ),
184+ Row (3 , " str3" ) :: Row (4 , " str4" ) :: Nil
185+ )
186+ sql(" DROP TABLE IF EXISTS test_insert_parquet" )
187+
188+ // Create it again.
189+ sql(
190+ """
191+ |create table test_insert_parquet
192+ |(
193+ | intField INT,
194+ | stringField STRING
195+ |)
196+ |ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
197+ |STORED AS
198+ | INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
199+ | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
200+ """ .stripMargin)
201+ // Insert overwrite an empty table.
202+ sql(" insert overwrite table test_insert_parquet select a, b from jt where jt.a < 5" )
203+ checkAnswer(
204+ sql(s " SELECT intField, stringField FROM test_insert_parquet WHERE intField > 2 " ),
205+ Row (3 , " str3" ) :: Row (4 , " str4" ) :: Nil
206+ )
207+ // Insert into the table.
208+ sql(" insert into table test_insert_parquet select a, b from jt" )
209+ checkAnswer(
210+ sql(s " SELECT intField, stringField FROM test_insert_parquet " ),
211+ (1 to 10 ).map(i => Row (i, s " str $i" )) ++ (1 to 4 ).map(i => Row (i, s " str $i" ))
212+ )
213+ sql(" DROP TABLE IF EXISTS test_insert_parquet" )
214+ }
215+
216+ test(" scan a parquet table created through a CTAS statement" ) {
164217 sql(
165218 """
166219 |create table test_parquet_ctas ROW FORMAT
@@ -183,10 +236,7 @@ class ParquetDataSourceOnMetastoreSuite extends ParquetMetastoreSuiteBase {
183236 s " test_parquet_ctas should be converted to ${classOf [ParquetRelation2 ].getCanonicalName}" )
184237 }
185238
186- sql(" DROP TABLE IF EXISTS jt" )
187239 sql(" DROP TABLE IF EXISTS test_parquet_ctas" )
188- setConf(" spark.sql.hive.convertMetastoreParquet" , originalConvertMetastore)
189- setConf(SQLConf .PARQUET_USE_DATA_SOURCE_API , originalUseDataSource)
190240 }
191241}
192242
0 commit comments