From 7c913b1d7cad7e137ad7f64c226814f6b5ab3649 Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Tue, 3 Mar 2015 00:09:42 +0800 Subject: [PATCH 1/2] Blacklists several test cases for Hive 0.12.0 --- .../sql/hive/MetastoreDataSourcesSuite.scala | 35 ++++++++++--------- .../sql/hive/execution/HiveQuerySuite.scala | 7 ++-- .../org/apache/spark/sql/hive/Shim12.scala | 13 ++++++- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 5d6a6f3b64f03..a09c0a8f465d8 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -64,25 +64,28 @@ class MetastoreDataSourcesSuite extends QueryTest with BeforeAndAfterEach { jsonFile(filePath).collect().toSeq) } - test ("persistent JSON table with a user specified schema") { - sql( - s""" - |CREATE TABLE jsonTable ( - |a string, - |b String, - |`c_!@(3)` int, - |`` Struct<`d!`:array, `=`:array>>) - |USING org.apache.spark.sql.json.DefaultSource - |OPTIONS ( - | path '${filePath}' - |) + // Hive 0.12.0 can't parse the SELECT statement used in this case. + if (HiveShim.version != "0.12.0") { + test ("persistent JSON table with a user specified schema") { + sql( + s""" + |CREATE TABLE jsonTable ( + |a string, + |b String, + |`c_!@(3)` int, + |`` Struct<`d!`:array, `=`:array>>) + |USING org.apache.spark.sql.json.DefaultSource + |OPTIONS ( + | path '${filePath}' + |) """.stripMargin) - jsonFile(filePath).registerTempTable("expectedJsonTable") + jsonFile(filePath).registerTempTable("expectedJsonTable") - checkAnswer( - sql("SELECT a, b, `c_!@(3)`, ``.`d!`, ``.`=` FROM jsonTable"), - sql("SELECT a, b, `c_!@(3)`, ``.`d!`, ``.`=` FROM expectedJsonTable").collect().toSeq) + checkAnswer( + sql("SELECT a, b, `c_!@(3)`, ``.`d!`, ``.`=` FROM jsonTable"), + sql("SELECT a, b, `c_!@(3)`, ``.`d!`, ``.`=` FROM expectedJsonTable").collect().toSeq) + } } test ("persistent JSON table with a user specified schema with a subset of fields") { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index de140fc72a2c3..13f2b3392a785 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -82,8 +82,11 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { printf("Bb%d", 12), "13", repeat(printf("s%d", 14), 2), "14") FROM src LIMIT 1""") - createQueryTest("NaN to Decimal", - "SELECT CAST(CAST('NaN' AS DOUBLE) AS DECIMAL(1,1)) FROM src LIMIT 1") + // Hive 0.12.0 doesn't support decimal with precision and scale. + if (HiveShim.version != "0.12.0") { + createQueryTest("NaN to Decimal", + "SELECT CAST(CAST('NaN' AS DOUBLE) AS DECIMAL(1,1)) FROM src LIMIT 1") + } createQueryTest("constant null testing", """SELECT diff --git a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala index 30646ddbc29d8..27de080e08970 100644 --- a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala +++ b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala @@ -220,7 +220,18 @@ private[hive] object HiveShim { "serde_regex", "udf_to_date", "udaf_collect_set", - "udf_concat" + "udf_concat", + + // The following 3 cases use 0.13.1 decimal syntax + "udf_to_double", + "udf_to_float", + "udf_pmod", + + // This case uses different ".q" files for 0.12.0 and 0.13.1. The version used in 0.13.1 + // includes a regression test for HIVE-4116, which wasn't fixed until 0.13.0. Developers who + // sets HIVE_DEV_HOME properly won't see any error. However, we don't have Hive source tree on + // Jenkins nodes, thus this case is blacklisted for the sake of Jenkins. + "create_view_translate" ) def setLocation(tbl: Table, crtTbl: CreateTableDesc): Unit = { From 074b88da44153cad849048df7da11d3f54895a6f Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Wed, 18 Mar 2015 22:07:06 +0800 Subject: [PATCH 2/2] Ignores some HiveThriftServer2 test cases when compiled against Hive 0.12.0 --- .../HiveThriftServer2Suites.scala | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index aff96e21a5373..25bc02f2dd759 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -105,11 +105,13 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { } } - test("Checks Hive version") { - withJdbcStatement { statement => - val resultSet = statement.executeQuery("SET spark.sql.hive.version") - resultSet.next() - assert(resultSet.getString(1) === s"spark.sql.hive.version=${HiveShim.version}") + if (HiveShim.version != "0.12.0") { + test("Checks Hive version") { + withJdbcStatement { statement => + val resultSet = statement.executeQuery("SET spark.sql.hive.version") + resultSet.next() + assert(resultSet.getString(1) === s"spark.sql.hive.version=${HiveShim.version}") + } } } @@ -340,30 +342,34 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { class HiveThriftHttpServerSuite extends HiveThriftJdbcTest { override def mode = ServerMode.http - test("JDBC query execution") { - withJdbcStatement { statement => - val queries = Seq( - "SET spark.sql.shuffle.partitions=3", - "DROP TABLE IF EXISTS test", - "CREATE TABLE test(key INT, val STRING)", - s"LOAD DATA LOCAL INPATH '${TestData.smallKv}' OVERWRITE INTO TABLE test", - "CACHE TABLE test") + // When compiled against Hive 0.12.0, HiveThriftServer2 HTTP mode doesn't work. Please refer to + // SPARK-6387 for details: https://issues.apache.org/jira/browse/SPARK-6387 + if (HiveShim.version != "0.12.0") { + test("JDBC query execution") { + withJdbcStatement { statement => + val queries = Seq( + "SET spark.sql.shuffle.partitions=3", + "DROP TABLE IF EXISTS test", + "CREATE TABLE test(key INT, val STRING)", + s"LOAD DATA LOCAL INPATH '${TestData.smallKv}' OVERWRITE INTO TABLE test", + "CACHE TABLE test") - queries.foreach(statement.execute) + queries.foreach(statement.execute) - assertResult(5, "Row count mismatch") { - val resultSet = statement.executeQuery("SELECT COUNT(*) FROM test") - resultSet.next() - resultSet.getInt(1) + assertResult(5, "Row count mismatch") { + val resultSet = statement.executeQuery("SELECT COUNT(*) FROM test") + resultSet.next() + resultSet.getInt(1) + } } } - } - test("Checks Hive version") { - withJdbcStatement { statement => - val resultSet = statement.executeQuery("SET spark.sql.hive.version") - resultSet.next() - assert(resultSet.getString(1) === s"spark.sql.hive.version=${HiveShim.version}") + test("Checks Hive version") { + withJdbcStatement { statement => + val resultSet = statement.executeQuery("SET spark.sql.hive.version") + resultSet.next() + assert(resultSet.getString(1) === s"spark.sql.hive.version=${HiveShim.version}") + } } } }