From 544573e7593c0df5630395cdd1725af2dfdb51ef Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Wed, 28 Jan 2015 18:48:15 +0800 Subject: [PATCH 01/15] initial --- .../org/apache/spark/sql/hive/HiveQl.scala | 18 ++++++++++-------- .../apache/spark/sql/hive/HiveStrategies.scala | 3 ++- .../sql/hive/execution/HiveQuerySuite.scala | 6 ++++++ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 5e29e57d9358..720d618936fc 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -54,13 +54,8 @@ private[hive] case object NativePlaceholder extends Command */ case class DescribeCommand( table: LogicalPlan, - isExtended: Boolean) extends Command { - override def output = Seq( - // Column names are based on Hive. - AttributeReference("col_name", StringType, nullable = false)(), - AttributeReference("data_type", StringType, nullable = false)(), - AttributeReference("comment", StringType, nullable = false)()) -} + override val output: Seq[Attribute], + isExtended: Boolean) extends Command /** Provides a mapping from HiveQL statements to catalyst logical plans and expression trees. */ private[hive] object HiveQl { @@ -509,7 +504,11 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C // TODO: Actually, a user may mean tableName.columnName. Need to resolve this issue. val tableIdent = extractTableIdent(nameParts.head) DescribeCommand( - UnresolvedRelation(tableIdent, None), extended.isDefined) + UnresolvedRelation(tableIdent, None), + Seq(AttributeReference("col_name", StringType, nullable = false)(), + AttributeReference("data_type", StringType, nullable = false)(), + AttributeReference("comment", StringType, nullable = false)()), + extended.isDefined) case Token(".", dbName :: tableName :: colName :: Nil) => // It is describing a column with the format like "describe db.table column". NativePlaceholder @@ -517,6 +516,9 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C // It is describing a table with the format like "describe table". DescribeCommand( UnresolvedRelation(Seq(tableName.getText), None), + Seq(AttributeReference("col_name", StringType, nullable = false)(), + AttributeReference("data_type", StringType, nullable = false)(), + AttributeReference("comment", StringType, nullable = false)()), extended.isDefined) } } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala index 6952b126cf89..6df049f6012d 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala @@ -223,7 +223,8 @@ private[hive] trait HiveStrategies { case class HiveCommandStrategy(context: HiveContext) extends Strategy { def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { case describe: DescribeCommand => - val resolvedTable = context.executePlan(describe.table).analyzed + val logical = describe.table + val resolvedTable = context.executePlan(logical).analyzed resolvedTable match { case t: MetastoreRelation => ExecutedCommand( diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index df72be7746ac..1f1789191f49 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -63,6 +63,12 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { } } + test("describe like query a table") { + loadTestTable("src") + sql("desc src").registerTempTable("mydesc") + assert(sql("select count(1) from mydesc").collect()(0)(0) == 2) + } + createQueryTest("! operator", """ |SELECT a FROM ( From ac2c3bbc725266204dacb142fdb5cbcdf6783311 Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Wed, 28 Jan 2015 20:03:35 +0800 Subject: [PATCH 02/15] refine code and test suite --- .../org/apache/spark/sql/hive/HiveQl.scala | 16 ++++------ .../spark/sql/hive/HiveStrategies.scala | 3 +- .../sql/hive/execution/HiveQuerySuite.scala | 29 +++++++++++++++++-- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 720d618936fc..ea1ceb5cab9a 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -488,6 +488,10 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C extended != None) case Token("TOK_DESCTABLE", describeArgs) => + val attrs = Seq(AttributeReference("col_name", StringType, nullable = false)(), + AttributeReference("data_type", StringType, nullable = false)(), + AttributeReference("comment", StringType, nullable = false)()) + // Reference: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL val Some(tableType) :: formatted :: extended :: pretty :: Nil = getClauses(Seq("TOK_TABTYPE", "FORMATTED", "EXTENDED", "PRETTY"), describeArgs) @@ -504,22 +508,14 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C // TODO: Actually, a user may mean tableName.columnName. Need to resolve this issue. val tableIdent = extractTableIdent(nameParts.head) DescribeCommand( - UnresolvedRelation(tableIdent, None), - Seq(AttributeReference("col_name", StringType, nullable = false)(), - AttributeReference("data_type", StringType, nullable = false)(), - AttributeReference("comment", StringType, nullable = false)()), - extended.isDefined) + UnresolvedRelation(tableIdent, None), attrs, extended.isDefined) case Token(".", dbName :: tableName :: colName :: Nil) => // It is describing a column with the format like "describe db.table column". NativePlaceholder case tableName => // It is describing a table with the format like "describe table". DescribeCommand( - UnresolvedRelation(Seq(tableName.getText), None), - Seq(AttributeReference("col_name", StringType, nullable = false)(), - AttributeReference("data_type", StringType, nullable = false)(), - AttributeReference("comment", StringType, nullable = false)()), - extended.isDefined) + UnresolvedRelation(Seq(tableName.getText), None), attrs, extended.isDefined) } } // All other cases. diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala index 6df049f6012d..6952b126cf89 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala @@ -223,8 +223,7 @@ private[hive] trait HiveStrategies { case class HiveCommandStrategy(context: HiveContext) extends Strategy { def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { case describe: DescribeCommand => - val logical = describe.table - val resolvedTable = context.executePlan(logical).analyzed + val resolvedTable = context.executePlan(describe.table).analyzed resolvedTable match { case t: MetastoreRelation => ExecutedCommand( diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 1f1789191f49..60d873742e51 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -63,10 +63,35 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { } } - test("describe like query a table") { + test("SPARK-5324 Results of describe can't be queried") { loadTestTable("src") + + // register a describe command to be a temp table sql("desc src").registerTempTable("mydesc") - assert(sql("select count(1) from mydesc").collect()(0)(0) == 2) + assertResult( + Array( + Row("col_name", "StringType", null), + Row("data_type", "StringType", null), + Row("comment", "StringType", null) + )) { + sql("desc mydesc").collect() + } + + assertResult( + Array( + Row("key", "int", null), + Row("value", "string", null) + )) { + sql("select * from mydesc").collect() + } + + assertResult( + Array( + Row("key", "int", null), + Row("value", "string", null) + )) { + sql("select col_name, data_type, comment from mydesc").collect() + } } createQueryTest("! operator", From aeaea5f3ed51d4dbef29f60f588dc01edc3e66e3 Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Thu, 29 Jan 2015 08:53:58 +0800 Subject: [PATCH 03/15] rename test suite --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 60d873742e51..080119c279c3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -56,14 +56,14 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { Locale.setDefault(originalLocale) } - test("SPARK-4908: concurent hive native commands") { + test("SPARK-4908: concurrent hive native commands") { (1 to 100).par.map { _ => sql("USE default") sql("SHOW TABLES") } } - test("SPARK-5324 Results of describe can't be queried") { + test("SPARK-5324 query result of describe command") { loadTestTable("src") // register a describe command to be a temp table From c5fdecfe3fbd4b3358aabd86d6efcf7bf0a0d91a Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Thu, 29 Jan 2015 08:55:23 +0800 Subject: [PATCH 04/15] code style --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 080119c279c3..3206ad481200 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -72,7 +72,7 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { Array( Row("col_name", "StringType", null), Row("data_type", "StringType", null), - Row("comment", "StringType", null) + Row("comment", "StringType", null) )) { sql("desc mydesc").collect() } From 11559ae5b8356e0b50b1647af1623b04ca42523a Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Thu, 29 Jan 2015 08:57:59 +0800 Subject: [PATCH 05/15] code style --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 3206ad481200..4be82b424016 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -72,7 +72,7 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { Array( Row("col_name", "StringType", null), Row("data_type", "StringType", null), - Row("comment", "StringType", null) + Row("comment", "StringType", null) )) { sql("desc mydesc").collect() } From f942c9b7b883deb026a74bdfc05f7cd89ef74111 Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Wed, 28 Jan 2015 18:48:15 +0800 Subject: [PATCH 06/15] initial --- .../org/apache/spark/sql/hive/HiveQl.scala | 18 ++++++++++-------- .../apache/spark/sql/hive/HiveStrategies.scala | 3 ++- .../sql/hive/execution/HiveQuerySuite.scala | 6 ++++++ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 30a64b48d795..90ad73768db5 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -54,13 +54,8 @@ private[hive] case object NativePlaceholder extends Command */ case class DescribeCommand( table: LogicalPlan, - isExtended: Boolean) extends Command { - override def output = Seq( - // Column names are based on Hive. - AttributeReference("col_name", StringType, nullable = false)(), - AttributeReference("data_type", StringType, nullable = false)(), - AttributeReference("comment", StringType, nullable = false)()) -} + override val output: Seq[Attribute], + isExtended: Boolean) extends Command /** Provides a mapping from HiveQL statements to catalyst logical plans and expression trees. */ private[hive] object HiveQl { @@ -509,7 +504,11 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C // TODO: Actually, a user may mean tableName.columnName. Need to resolve this issue. val tableIdent = extractTableIdent(nameParts.head) DescribeCommand( - UnresolvedRelation(tableIdent, None), extended.isDefined) + UnresolvedRelation(tableIdent, None), + Seq(AttributeReference("col_name", StringType, nullable = false)(), + AttributeReference("data_type", StringType, nullable = false)(), + AttributeReference("comment", StringType, nullable = false)()), + extended.isDefined) case Token(".", dbName :: tableName :: colName :: Nil) => // It is describing a column with the format like "describe db.table column". NativePlaceholder @@ -517,6 +516,9 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C // It is describing a table with the format like "describe table". DescribeCommand( UnresolvedRelation(Seq(tableName.getText), None), + Seq(AttributeReference("col_name", StringType, nullable = false)(), + AttributeReference("data_type", StringType, nullable = false)(), + AttributeReference("comment", StringType, nullable = false)()), extended.isDefined) } } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala index ace9329cd582..58c1d3353da0 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala @@ -222,7 +222,8 @@ private[hive] trait HiveStrategies { case class HiveCommandStrategy(context: HiveContext) extends Strategy { def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { case describe: DescribeCommand => - val resolvedTable = context.executePlan(describe.table).analyzed + val logical = describe.table + val resolvedTable = context.executePlan(logical).analyzed resolvedTable match { case t: MetastoreRelation => ExecutedCommand( diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 60619f5d9957..02f2b67b4fbb 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -64,6 +64,12 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { } } + test("describe like query a table") { + loadTestTable("src") + sql("desc src").registerTempTable("mydesc") + assert(sql("select count(1) from mydesc").collect()(0)(0) == 2) + } + createQueryTest("! operator", """ |SELECT a FROM ( From 75f2342fda7662b2b1503df78980fbf4bcf16b5a Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Wed, 28 Jan 2015 20:03:35 +0800 Subject: [PATCH 07/15] refine code and test suite --- .../org/apache/spark/sql/hive/HiveQl.scala | 16 ++++------ .../spark/sql/hive/HiveStrategies.scala | 3 +- .../sql/hive/execution/HiveQuerySuite.scala | 29 +++++++++++++++++-- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 90ad73768db5..44f6bcebddea 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -488,6 +488,10 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C extended != None) case Token("TOK_DESCTABLE", describeArgs) => + val attrs = Seq(AttributeReference("col_name", StringType, nullable = false)(), + AttributeReference("data_type", StringType, nullable = false)(), + AttributeReference("comment", StringType, nullable = false)()) + // Reference: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL val Some(tableType) :: formatted :: extended :: pretty :: Nil = getClauses(Seq("TOK_TABTYPE", "FORMATTED", "EXTENDED", "PRETTY"), describeArgs) @@ -504,22 +508,14 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C // TODO: Actually, a user may mean tableName.columnName. Need to resolve this issue. val tableIdent = extractTableIdent(nameParts.head) DescribeCommand( - UnresolvedRelation(tableIdent, None), - Seq(AttributeReference("col_name", StringType, nullable = false)(), - AttributeReference("data_type", StringType, nullable = false)(), - AttributeReference("comment", StringType, nullable = false)()), - extended.isDefined) + UnresolvedRelation(tableIdent, None), attrs, extended.isDefined) case Token(".", dbName :: tableName :: colName :: Nil) => // It is describing a column with the format like "describe db.table column". NativePlaceholder case tableName => // It is describing a table with the format like "describe table". DescribeCommand( - UnresolvedRelation(Seq(tableName.getText), None), - Seq(AttributeReference("col_name", StringType, nullable = false)(), - AttributeReference("data_type", StringType, nullable = false)(), - AttributeReference("comment", StringType, nullable = false)()), - extended.isDefined) + UnresolvedRelation(Seq(tableName.getText), None), attrs, extended.isDefined) } } // All other cases. diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala index 58c1d3353da0..ace9329cd582 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala @@ -222,8 +222,7 @@ private[hive] trait HiveStrategies { case class HiveCommandStrategy(context: HiveContext) extends Strategy { def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { case describe: DescribeCommand => - val logical = describe.table - val resolvedTable = context.executePlan(logical).analyzed + val resolvedTable = context.executePlan(describe.table).analyzed resolvedTable match { case t: MetastoreRelation => ExecutedCommand( diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 02f2b67b4fbb..dab7a27c2b71 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -64,10 +64,35 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { } } - test("describe like query a table") { + test("SPARK-5324 Results of describe can't be queried") { loadTestTable("src") + + // register a describe command to be a temp table sql("desc src").registerTempTable("mydesc") - assert(sql("select count(1) from mydesc").collect()(0)(0) == 2) + assertResult( + Array( + Row("col_name", "StringType", null), + Row("data_type", "StringType", null), + Row("comment", "StringType", null) + )) { + sql("desc mydesc").collect() + } + + assertResult( + Array( + Row("key", "int", null), + Row("value", "string", null) + )) { + sql("select * from mydesc").collect() + } + + assertResult( + Array( + Row("key", "int", null), + Row("value", "string", null) + )) { + sql("select col_name, data_type, comment from mydesc").collect() + } } createQueryTest("! operator", From c7d606d8f7b372f81d612295d5c7ff4a6135d679 Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Thu, 29 Jan 2015 08:53:58 +0800 Subject: [PATCH 08/15] rename test suite --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index dab7a27c2b71..a105dab6dfc0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -57,14 +57,14 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { Locale.setDefault(originalLocale) } - test("SPARK-4908: concurent hive native commands") { + test("SPARK-4908: concurrent hive native commands") { (1 to 100).par.map { _ => sql("USE default") sql("SHOW TABLES") } } - test("SPARK-5324 Results of describe can't be queried") { + test("SPARK-5324 query result of describe command") { loadTestTable("src") // register a describe command to be a temp table From dd0aaef9bb24a3fc972fb1dc25a278fb4092c4f4 Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Thu, 29 Jan 2015 08:55:23 +0800 Subject: [PATCH 09/15] code style --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index a105dab6dfc0..1dd8bc383055 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -73,7 +73,7 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { Array( Row("col_name", "StringType", null), Row("data_type", "StringType", null), - Row("comment", "StringType", null) + Row("comment", "StringType", null) )) { sql("desc mydesc").collect() } From 0015f8292a66cd6eef72c90b704c47a854e1662c Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Thu, 29 Jan 2015 08:57:59 +0800 Subject: [PATCH 10/15] code style --- .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 1dd8bc383055..f23949a06b11 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -73,7 +73,7 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { Array( Row("col_name", "StringType", null), Row("data_type", "StringType", null), - Row("comment", "StringType", null) + Row("comment", "StringType", null) )) { sql("desc mydesc").collect() } From e1da481ab42a450e7873b42d810f325148729867 Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Tue, 3 Feb 2015 00:09:28 +0800 Subject: [PATCH 11/15] refine test suite --- .../HiveOperatorQueryableSuite.scala | 48 +++++++++++++++++++ .../sql/hive/execution/HiveQuerySuite.scala | 31 ------------ 2 files changed, 48 insertions(+), 31 deletions(-) create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala new file mode 100644 index 000000000000..bcf5c2930023 --- /dev/null +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.execution + +import org.apache.spark.sql.{Row, QueryTest} +import org.apache.spark.sql.hive.test.TestHive._ + +/** + * A set of tests that validates commands can also be queried by like a table + */ +class HiveOperatorQueryableSuite extends QueryTest { + test("SPARK-5324 query result of describe command") { + loadTestTable("src") + + // register a describe command to be a temp table + sql("desc src").registerTempTable("mydesc") + checkAnswer( + sql("desc mydesc"), + Seq(Row("col_name", "StringType", null), + Row("data_type", "StringType", null), + Row("comment", "StringType", null))) + + checkAnswer( + sql("select * from mydesc"), + Seq(Row("key", "int", null), + Row("value", "string", null))) + + checkAnswer( + sql("select col_name, data_type, comment from mydesc"), + Seq(Row("key", "int", null), + Row("value", "string", null))) + } +} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index f23949a06b11..215f7c5cb9f3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -64,37 +64,6 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { } } - test("SPARK-5324 query result of describe command") { - loadTestTable("src") - - // register a describe command to be a temp table - sql("desc src").registerTempTable("mydesc") - assertResult( - Array( - Row("col_name", "StringType", null), - Row("data_type", "StringType", null), - Row("comment", "StringType", null) - )) { - sql("desc mydesc").collect() - } - - assertResult( - Array( - Row("key", "int", null), - Row("value", "string", null) - )) { - sql("select * from mydesc").collect() - } - - assertResult( - Array( - Row("key", "int", null), - Row("value", "string", null) - )) { - sql("select col_name, data_type, comment from mydesc").collect() - } - } - createQueryTest("! operator", """ |SELECT a FROM ( From d541a3508f296b53dc354e3a309ca15e5bfcc473 Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Tue, 3 Feb 2015 00:11:08 +0800 Subject: [PATCH 12/15] refine test suite --- sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 44f6bcebddea..93d5b8f53dd4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -55,7 +55,7 @@ private[hive] case object NativePlaceholder extends Command case class DescribeCommand( table: LogicalPlan, override val output: Seq[Attribute], - isExtended: Boolean) extends Command + isExtended: Boolean) extends Command /** Provides a mapping from HiveQL statements to catalyst logical plans and expression trees. */ private[hive] object HiveQl { From 354ad715a82acb54d1bc6def314f75ef7d55b1f4 Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Tue, 3 Feb 2015 00:13:11 +0800 Subject: [PATCH 13/15] query describe command --- .../org/apache/spark/sql/hive/HiveQl.scala | 18 ++++--- .../HiveOperatorQueryableSuite.scala | 48 +++++++++++++++++++ .../sql/hive/execution/HiveQuerySuite.scala | 2 +- 3 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 30a64b48d795..93d5b8f53dd4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -54,13 +54,8 @@ private[hive] case object NativePlaceholder extends Command */ case class DescribeCommand( table: LogicalPlan, - isExtended: Boolean) extends Command { - override def output = Seq( - // Column names are based on Hive. - AttributeReference("col_name", StringType, nullable = false)(), - AttributeReference("data_type", StringType, nullable = false)(), - AttributeReference("comment", StringType, nullable = false)()) -} + override val output: Seq[Attribute], + isExtended: Boolean) extends Command /** Provides a mapping from HiveQL statements to catalyst logical plans and expression trees. */ private[hive] object HiveQl { @@ -493,6 +488,10 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C extended != None) case Token("TOK_DESCTABLE", describeArgs) => + val attrs = Seq(AttributeReference("col_name", StringType, nullable = false)(), + AttributeReference("data_type", StringType, nullable = false)(), + AttributeReference("comment", StringType, nullable = false)()) + // Reference: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL val Some(tableType) :: formatted :: extended :: pretty :: Nil = getClauses(Seq("TOK_TABTYPE", "FORMATTED", "EXTENDED", "PRETTY"), describeArgs) @@ -509,15 +508,14 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C // TODO: Actually, a user may mean tableName.columnName. Need to resolve this issue. val tableIdent = extractTableIdent(nameParts.head) DescribeCommand( - UnresolvedRelation(tableIdent, None), extended.isDefined) + UnresolvedRelation(tableIdent, None), attrs, extended.isDefined) case Token(".", dbName :: tableName :: colName :: Nil) => // It is describing a column with the format like "describe db.table column". NativePlaceholder case tableName => // It is describing a table with the format like "describe table". DescribeCommand( - UnresolvedRelation(Seq(tableName.getText), None), - extended.isDefined) + UnresolvedRelation(Seq(tableName.getText), None), attrs, extended.isDefined) } } // All other cases. diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala new file mode 100644 index 000000000000..bcf5c2930023 --- /dev/null +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.execution + +import org.apache.spark.sql.{Row, QueryTest} +import org.apache.spark.sql.hive.test.TestHive._ + +/** + * A set of tests that validates commands can also be queried by like a table + */ +class HiveOperatorQueryableSuite extends QueryTest { + test("SPARK-5324 query result of describe command") { + loadTestTable("src") + + // register a describe command to be a temp table + sql("desc src").registerTempTable("mydesc") + checkAnswer( + sql("desc mydesc"), + Seq(Row("col_name", "StringType", null), + Row("data_type", "StringType", null), + Row("comment", "StringType", null))) + + checkAnswer( + sql("select * from mydesc"), + Seq(Row("key", "int", null), + Row("value", "string", null))) + + checkAnswer( + sql("select col_name, data_type, comment from mydesc"), + Seq(Row("key", "int", null), + Row("value", "string", null))) + } +} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 60619f5d9957..215f7c5cb9f3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -57,7 +57,7 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { Locale.setDefault(originalLocale) } - test("SPARK-4908: concurent hive native commands") { + test("SPARK-4908: concurrent hive native commands") { (1 to 100).par.map { _ => sql("USE default") sql("SHOW TABLES") From 3ba10580f9ba951a3e9627befdc6dd5408795b8c Mon Sep 17 00:00:00 2001 From: OopsOutOfMemory Date: Wed, 4 Feb 2015 23:22:44 +0800 Subject: [PATCH 14/15] change to default argument --- .../scala/org/apache/spark/sql/hive/HiveQl.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 522906e0c12e..49d738509c2a 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -55,7 +55,10 @@ private[hive] case object NativePlaceholder extends Command */ case class DescribeCommand( table: LogicalPlan, - override val output: Seq[Attribute], + override val output: Seq[Attribute] = + Seq(AttributeReference("col_name", StringType, nullable = false)(), + AttributeReference("data_type", StringType, nullable = false)(), + AttributeReference("comment", StringType, nullable = false)()), isExtended: Boolean) extends Command /** Provides a mapping from HiveQL statements to catalyst logical plans and expression trees. */ @@ -489,10 +492,6 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C extended != None) case Token("TOK_DESCTABLE", describeArgs) => - val attrs = Seq(AttributeReference("col_name", StringType, nullable = false)(), - AttributeReference("data_type", StringType, nullable = false)(), - AttributeReference("comment", StringType, nullable = false)()) - // Reference: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL val Some(tableType) :: formatted :: extended :: pretty :: Nil = getClauses(Seq("TOK_TABTYPE", "FORMATTED", "EXTENDED", "PRETTY"), describeArgs) @@ -509,14 +508,14 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C // TODO: Actually, a user may mean tableName.columnName. Need to resolve this issue. val tableIdent = extractTableIdent(nameParts.head) DescribeCommand( - UnresolvedRelation(tableIdent, None), attrs, extended.isDefined) + UnresolvedRelation(tableIdent, None), isExtended = extended.isDefined) case Token(".", dbName :: tableName :: colName :: Nil) => // It is describing a column with the format like "describe db.table column". NativePlaceholder case tableName => // It is describing a table with the format like "describe table". DescribeCommand( - UnresolvedRelation(Seq(tableName.getText), None), attrs, extended.isDefined) + UnresolvedRelation(Seq(tableName.getText), None), isExtended = extended.isDefined) } } // All other cases. From e71430a5709836e74556713ec5644f54abc032f9 Mon Sep 17 00:00:00 2001 From: "Sheng, Li" Date: Thu, 5 Feb 2015 15:34:50 +0800 Subject: [PATCH 15/15] Update HiveOperatorQueryableSuite.scala style fix --- .../sql/hive/execution/HiveOperatorQueryableSuite.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala index bcf5c2930023..1f00651e7dba 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala @@ -31,18 +31,21 @@ class HiveOperatorQueryableSuite extends QueryTest { sql("desc src").registerTempTable("mydesc") checkAnswer( sql("desc mydesc"), - Seq(Row("col_name", "StringType", null), + Seq( + Row("col_name", "StringType", null), Row("data_type", "StringType", null), Row("comment", "StringType", null))) checkAnswer( sql("select * from mydesc"), - Seq(Row("key", "int", null), + Seq( + Row("key", "int", null), Row("value", "string", null))) checkAnswer( sql("select col_name, data_type, comment from mydesc"), - Seq(Row("key", "int", null), + Seq( + Row("key", "int", null), Row("value", "string", null))) } }