From efb0b4bfc5643b5cd513682b035f24b5fa9656e9 Mon Sep 17 00:00:00 2001 From: cenyuhai <261810726@qq.com> Date: Tue, 7 Feb 2017 19:33:15 +0800 Subject: [PATCH 1/8] change column names to lower case --- .../scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 208c8c9d5d0cf..69184a88a22f8 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -980,9 +980,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat predicates: Seq[Expression]): Seq[CatalogTablePartition] = withClient { val rawTable = getRawTable(db, table) val catalogTable = restoreTableMetadata(rawTable) - val partitionColumnNames = catalogTable.partitionColumnNames.toSet + val partitionColumnNames = catalogTable.partitionColumnNames.map(_.toLowerCase).toSet val nonPartitionPruningPredicates = predicates.filterNot { - _.references.map(_.name).toSet.subsetOf(partitionColumnNames) + _.references.map(_.name.toLowerCase).toSet.subsetOf(partitionColumnNames) } if (nonPartitionPruningPredicates.nonEmpty) { From 6884ad35ad2f6feb898c92f55d7840972b1cb3df Mon Sep 17 00:00:00 2001 From: cenyuhai <261810726@qq.com> Date: Fri, 10 Feb 2017 14:56:10 +0800 Subject: [PATCH 2/8] change hive column names to lower case --- .../scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 69184a88a22f8..b4ba59dac4d86 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -1000,7 +1000,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat val boundPredicate = InterpretedPredicate.create(predicates.reduce(And).transform { case att: AttributeReference => - val index = partitionSchema.indexWhere(_.name == att.name) + val index = partitionSchema.indexWhere(_.name.toLowerCase == att.name.toLowerCase) BoundReference(index, partitionSchema(index).dataType, nullable = true) }) clientPrunedPartitions.filter { p => boundPredicate(p.toRow(partitionSchema)) } From 7214b8a9c3fd7b5d4c352b3dd908f54089a68bc1 Mon Sep 17 00:00:00 2001 From: cenyuhai <261810726@qq.com> Date: Sun, 12 Feb 2017 10:42:22 +0800 Subject: [PATCH 3/8] add test case --- .../apache/spark/sql/hive/execution/HiveQuerySuite.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index dd278f683a3cd..c7525ba64f701 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1236,6 +1236,12 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd } } } + + createQueryTest("SPARK-19490: case sensitivity when filtering hive partition columns", + """ + | CREATE TABLE partition_test (key Int) partitioned by date; + | SELECT * FROM partition_test where DATE = '20170101'; + """.stripMargin) } // for SPARK-2180 test From b7346460babc8d1c9422ff8a0acf969e5b180568 Mon Sep 17 00:00:00 2001 From: cenyuhai <261810726@qq.com> Date: Sun, 12 Feb 2017 12:40:12 +0800 Subject: [PATCH 4/8] fix runtime error --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index c7525ba64f701..1661225752f04 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1237,7 +1237,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd } } - createQueryTest("SPARK-19490: case sensitivity when filtering hive partition columns", + createQueryTest("SPARK-19490 case sensitivity when filtering hive partition columns", """ | CREATE TABLE partition_test (key Int) partitioned by date; | SELECT * FROM partition_test where DATE = '20170101'; From 358769c6a5bedbbe4e94a940d55896e6a1b74096 Mon Sep 17 00:00:00 2001 From: cenyuhai <261810726@qq.com> Date: Sun, 12 Feb 2017 12:59:37 +0800 Subject: [PATCH 5/8] fix error in create table syntax --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 1661225752f04..4736138d5ad02 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1239,7 +1239,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd createQueryTest("SPARK-19490 case sensitivity when filtering hive partition columns", """ - | CREATE TABLE partition_test (key Int) partitioned by date; + | CREATE TABLE partition_test (key Int) partitioned by (date string); | SELECT * FROM partition_test where DATE = '20170101'; """.stripMargin) } From 5e91877ba0e14faa2920f4f4121be361a78b685f Mon Sep 17 00:00:00 2001 From: cenyuhai <261810726@qq.com> Date: Sun, 12 Feb 2017 14:48:32 +0800 Subject: [PATCH 6/8] insert data for test case --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 4736138d5ad02..102f1ac06cbd8 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1240,6 +1240,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd createQueryTest("SPARK-19490 case sensitivity when filtering hive partition columns", """ | CREATE TABLE partition_test (key Int) partitioned by (date string); + | INSERT OVERWRITE TABLE partition_test partition('20170101') select 1 from src limit 1; | SELECT * FROM partition_test where DATE = '20170101'; """.stripMargin) } From f7908219735137fe5402201f6cad45fa279f3f5f Mon Sep 17 00:00:00 2001 From: cenyuhai <261810726@qq.com> Date: Wed, 22 Feb 2017 15:48:03 +0800 Subject: [PATCH 7/8] fix bug in test case --- .../spark/sql/hive/execution/HiveQuerySuite.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 102f1ac06cbd8..d1936375295c8 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1237,12 +1237,11 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd } } - createQueryTest("SPARK-19490 case sensitivity when filtering hive partition columns", - """ - | CREATE TABLE partition_test (key Int) partitioned by (date string); - | INSERT OVERWRITE TABLE partition_test partition('20170101') select 1 from src limit 1; - | SELECT * FROM partition_test where DATE = '20170101'; - """.stripMargin) + test("SPARK-19490 case sensitivity when filtering hive partition columns") { + sql("CREATE TABLE partition_test (key Int) partitioned by (date string)") + sql("INSERT OVERWRITE TABLE partition_test partition('20170101') select 1 from src limit 1") + sql("SELECT * FROM partition_test where DATE = '20170101'") + } } // for SPARK-2180 test From c9dd09b92b035eec2ba5474945c9d3b4c0cc9650 Mon Sep 17 00:00:00 2001 From: cenyuhai <261810726@qq.com> Date: Wed, 22 Feb 2017 21:15:54 +0800 Subject: [PATCH 8/8] fix bug in test case --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index d1936375295c8..4c1640cf3e448 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1239,7 +1239,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd test("SPARK-19490 case sensitivity when filtering hive partition columns") { sql("CREATE TABLE partition_test (key Int) partitioned by (date string)") - sql("INSERT OVERWRITE TABLE partition_test partition('20170101') select 1 from src limit 1") + sql("INSERT INTO TABLE partition_test partition(date='20170101') select 1 from src limit 1") sql("SELECT * FROM partition_test where DATE = '20170101'") } }