From e9604a080c4fc31f333c7bb10eed9fca13460cf0 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Thu, 2 Jul 2020 17:42:19 +0800 Subject: [PATCH 1/4] temp modify --- .../sql/hive/HiveExternalCatalogSuite.scala | 53 +++++++------------ 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala index 473a93bf129d..689cad0cbda6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala @@ -182,40 +182,23 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite { } test("SPARK-31061: alterTable should be able to change table provider") { - val catalog = newBasicCatalog() - val parquetTable = CatalogTable( - identifier = TableIdentifier("parq_tbl", Some("db1")), - tableType = CatalogTableType.MANAGED, - storage = storageFormat.copy(locationUri = Some(new URI("file:/some/path"))), - schema = new StructType().add("col1", "int").add("col2", "string"), - provider = Some("parquet")) - catalog.createTable(parquetTable, ignoreIfExists = false) - - val rawTable = externalCatalog.getTable("db1", "parq_tbl") - assert(rawTable.provider === Some("parquet")) - - val fooTable = parquetTable.copy(provider = Some("foo")) - catalog.alterTable(fooTable) - val alteredTable = externalCatalog.getTable("db1", "parq_tbl") - assert(alteredTable.provider === Some("foo")) - } - - test("SPARK-31061: alterTable should be able to change table provider from hive") { - val catalog = newBasicCatalog() - val hiveTable = CatalogTable( - identifier = TableIdentifier("parq_tbl", Some("db1")), - tableType = CatalogTableType.MANAGED, - storage = storageFormat, - schema = new StructType().add("col1", "int").add("col2", "string"), - provider = Some("hive")) - catalog.createTable(hiveTable, ignoreIfExists = false) - - val rawTable = externalCatalog.getTable("db1", "parq_tbl") - assert(rawTable.provider === Some("hive")) - - val fooTable = rawTable.copy(provider = Some("foo")) - catalog.alterTable(fooTable) - val alteredTable = externalCatalog.getTable("db1", "parq_tbl") - assert(alteredTable.provider === Some("foo")) + Seq("parquet", "hive").foreach(i => { + val catalog = newBasicCatalog() + val parquetTable = CatalogTable( + identifier = TableIdentifier("parq_tbl", Some("db1")), + tableType = CatalogTableType.MANAGED, + storage = storageFormat, + schema = new StructType().add("col1", "int"), + provider = Some(i)) + catalog.createTable(parquetTable, ignoreIfExists = false) + + val rawTable = externalCatalog.getTable("db1", "parq_tbl") + assert(rawTable.provider === Some(i)) + + val fooTable = parquetTable.copy(provider = Some("foo")) + catalog.alterTable(fooTable) + val alteredTable = externalCatalog.getTable("db1", "parq_tbl") + assert(alteredTable.provider === Some("foo")) + }) } } From 17f39283f6916d61aef29e07f755dd39d1a75dd3 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Thu, 2 Jul 2020 17:49:58 +0800 Subject: [PATCH 2/4] correct test case and polish --- .../apache/spark/sql/hive/HiveExternalCatalogSuite.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala index 689cad0cbda6..c7ba6642f079 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala @@ -181,21 +181,22 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite { "INSERT overwrite directory \"fs://localhost/tmp\" select 1 as a")) } - test("SPARK-31061: alterTable should be able to change table provider") { + test("SPARK-31061: alterTable should be able to change table provider/hive") { + val catalog = newBasicCatalog() Seq("parquet", "hive").foreach(i => { - val catalog = newBasicCatalog() val parquetTable = CatalogTable( identifier = TableIdentifier("parq_tbl", Some("db1")), tableType = CatalogTableType.MANAGED, storage = storageFormat, schema = new StructType().add("col1", "int"), provider = Some(i)) + catalog.dropTable("db1", "parq_tbl", true, true) catalog.createTable(parquetTable, ignoreIfExists = false) val rawTable = externalCatalog.getTable("db1", "parq_tbl") assert(rawTable.provider === Some(i)) - val fooTable = parquetTable.copy(provider = Some("foo")) + val fooTable = rawTable.copy(provider = Some("foo")) catalog.alterTable(fooTable) val alteredTable = externalCatalog.getTable("db1", "parq_tbl") assert(alteredTable.provider === Some("foo")) From 0fb7829b9d632d2ee4888883bc3f56855883efaa Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Thu, 2 Jul 2020 18:09:24 +0800 Subject: [PATCH 3/4] rename parquetTable to tableDDL to compatible with parquet and hive --- .../org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala index c7ba6642f079..ed052d6197bb 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala @@ -184,14 +184,14 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite { test("SPARK-31061: alterTable should be able to change table provider/hive") { val catalog = newBasicCatalog() Seq("parquet", "hive").foreach(i => { - val parquetTable = CatalogTable( + val tableDDL = CatalogTable( identifier = TableIdentifier("parq_tbl", Some("db1")), tableType = CatalogTableType.MANAGED, storage = storageFormat, schema = new StructType().add("col1", "int"), provider = Some(i)) catalog.dropTable("db1", "parq_tbl", true, true) - catalog.createTable(parquetTable, ignoreIfExists = false) + catalog.createTable(tableDDL, ignoreIfExists = false) val rawTable = externalCatalog.getTable("db1", "parq_tbl") assert(rawTable.provider === Some(i)) From 899780e2f9f03d5d6a6a4b7071c246cd7a3eb938 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Thu, 2 Jul 2020 19:31:01 +0800 Subject: [PATCH 4/4] i => provider --- .../apache/spark/sql/hive/HiveExternalCatalogSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala index ed052d6197bb..270595b0011e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala @@ -183,18 +183,18 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite { test("SPARK-31061: alterTable should be able to change table provider/hive") { val catalog = newBasicCatalog() - Seq("parquet", "hive").foreach(i => { + Seq("parquet", "hive").foreach( provider => { val tableDDL = CatalogTable( identifier = TableIdentifier("parq_tbl", Some("db1")), tableType = CatalogTableType.MANAGED, storage = storageFormat, schema = new StructType().add("col1", "int"), - provider = Some(i)) + provider = Some(provider)) catalog.dropTable("db1", "parq_tbl", true, true) catalog.createTable(tableDDL, ignoreIfExists = false) val rawTable = externalCatalog.getTable("db1", "parq_tbl") - assert(rawTable.provider === Some(i)) + assert(rawTable.provider === Some(provider)) val fooTable = rawTable.copy(provider = Some("foo")) catalog.alterTable(fooTable)