From 2ebeac854144aa4a036cac9e309c5927677b6656 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Wed, 22 Mar 2017 19:41:49 -0700 Subject: [PATCH 01/15] fix. --- .../org/apache/spark/sql/catalyst/catalog/interface.scala | 2 +- .../main/scala/org/apache/spark/sql/types/StructType.scala | 6 ++++-- .../src/test/resources/sql-tests/results/describe.sql.out | 2 +- .../test/resources/sql-tests/results/show-tables.sql.out | 6 +++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 70ed44e025f5..b255e32fe481 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -283,7 +283,7 @@ case class CatalogTable( s"Created: ${new Date(createTime).toString}", s"Last Access: ${new Date(lastAccessTime).toString}", s"Type: ${tableType.name}", - if (schema.nonEmpty) s"Schema: ${schema.mkString("[", ", ", "]")}" else "", + if (schema.nonEmpty) s"Schema: ${schema.sql}" else "", if (provider.isDefined) s"Provider: ${provider.get}" else "", if (partitionColumnNames.nonEmpty) s"Partition Columns: $partitionColumns" else "" ) ++ bucketStrings ++ Seq( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala index 8d8b5b86d5aa..b4e28e302e5d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala @@ -343,7 +343,9 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru } override def sql: String = { - val fieldTypes = fields.map(f => s"${quoteIdentifier(f.name)}: ${f.dataType.sql}") + val fieldTypes = fields.map { f => + s"${quoteIdentifier(f.name)}: ${f.dataType.sql} (nullable = ${f.nullable})" + } s"STRUCT<${fieldTypes.mkString(", ")}>" } @@ -380,7 +382,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru override private[spark] def asNullable: StructType = { val newFields = fields.map { - case StructField(name, dataType, nullable, metadata) => + case StructField(name, dataType, _, metadata) => StructField(name, dataType.asNullable, nullable = true, metadata) } diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 422d548ea8de..f1de975c44d5 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -101,7 +101,7 @@ struct Created: Last Access: Type: MANAGED - Schema: [StructField(a,StringType,true), StructField(b,IntegerType,true), StructField(c,StringType,true), StructField(d,StringType,true)] + Schema: STRUCT<`a`: STRING (nullable = true), `b`: INT (nullable = true), `c`: STRING (nullable = true), `d`: STRING (nullable = true)> Provider: parquet Partition Columns: [`c`, `d`] Comment: table_comment diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out index 6d62e6092147..82c6225f4d84 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out @@ -123,7 +123,7 @@ show_t3 true CatalogTable( Created: Last Access: Type: VIEW - Schema: [StructField(e,IntegerType,true)] + Schema: STRUCT<`e`: INT (nullable = true)> Storage()) showdb show_t1 false CatalogTable( @@ -131,7 +131,7 @@ showdb show_t1 false CatalogTable( Created: Last Access: Type: MANAGED - Schema: [StructField(a,StringType,true), StructField(b,IntegerType,true), StructField(c,StringType,true), StructField(d,StringType,true)] + Schema: STRUCT<`a`: STRING (nullable = true), `b`: INT (nullable = true), `c`: STRING (nullable = true), `d`: STRING (nullable = true)> Provider: parquet Partition Columns: [`c`, `d`] Storage(Location: sql/core/spark-warehouse/showdb.db/show_t1) @@ -142,7 +142,7 @@ showdb show_t2 false CatalogTable( Created: Last Access: Type: MANAGED - Schema: [StructField(b,StringType,true), StructField(d,IntegerType,true)] + Schema: STRUCT<`b`: STRING (nullable = true), `d`: INT (nullable = true)> Provider: parquet Storage(Location: sql/core/spark-warehouse/showdb.db/show_t2)) From 87209193557d363412bf4041cddeb86d60affaf4 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Wed, 22 Mar 2017 22:39:24 -0700 Subject: [PATCH 02/15] improve --- .../sql/catalyst/catalog/interface.scala | 6 +-- .../sql-tests/results/describe.sql.out | 25 +++++---- .../sql-tests/results/show-tables.sql.out | 54 +++++++++++-------- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index b255e32fe481..5d19702e300c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -283,7 +283,6 @@ case class CatalogTable( s"Created: ${new Date(createTime).toString}", s"Last Access: ${new Date(lastAccessTime).toString}", s"Type: ${tableType.name}", - if (schema.nonEmpty) s"Schema: ${schema.sql}" else "", if (provider.isDefined) s"Provider: ${provider.get}" else "", if (partitionColumnNames.nonEmpty) s"Partition Columns: $partitionColumns" else "" ) ++ bucketStrings ++ Seq( @@ -292,9 +291,10 @@ case class CatalogTable( if (properties.nonEmpty) s"Properties: $tableProperties" else "", if (stats.isDefined) s"Statistics: ${stats.get.simpleString}" else "", s"$storage", - if (tracksPartitionsInCatalog) "Partition Provider: Catalog" else "") + if (tracksPartitionsInCatalog) "Partition Provider: Catalog" else "", + if (schema.nonEmpty) s"Schema: ${schema.treeString}" else "") - output.filter(_.nonEmpty).mkString("CatalogTable(\n\t", "\n\t", ")") + output.filter(_.nonEmpty).mkString("CatalogTable(\n", "\n", ")") } } diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index f1de975c44d5..1d9370f2869d 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -97,16 +97,21 @@ DESC EXTENDED t struct -- !query 6 output # Detailed Table Information CatalogTable( - Table: `default`.`t` - Created: - Last Access: - Type: MANAGED - Schema: STRUCT<`a`: STRING (nullable = true), `b`: INT (nullable = true), `c`: STRING (nullable = true), `d`: STRING (nullable = true)> - Provider: parquet - Partition Columns: [`c`, `d`] - Comment: table_comment - Storage(Location: sql/core/spark-warehouse/t) - Partition Provider: Catalog) +Table: `default`.`t` +Created: +Last Access: +Type: MANAGED +Provider: parquet +Partition Columns: [`c`, `d`] +Comment: table_comment +Storage(Location: sql/core/spark-warehouse/t) +Partition Provider: Catalog +Schema: root + |-- a: string (nullable = true) + |-- b: integer (nullable = true) + |-- c: string (nullable = true) + |-- d: string (nullable = true) +) # Partition Information # col_name data_type comment a string diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out index 82c6225f4d84..9d52caedddb8 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out @@ -119,32 +119,42 @@ SHOW TABLE EXTENDED LIKE 'show_t*' struct -- !query 12 output show_t3 true CatalogTable( - Table: `show_t3` - Created: - Last Access: - Type: VIEW - Schema: STRUCT<`e`: INT (nullable = true)> - Storage()) +Table: `show_t3` +Created: +Last Access: +Type: VIEW +Storage() +Schema: root + |-- e: integer (nullable = true) +) showdb show_t1 false CatalogTable( - Table: `showdb`.`show_t1` - Created: - Last Access: - Type: MANAGED - Schema: STRUCT<`a`: STRING (nullable = true), `b`: INT (nullable = true), `c`: STRING (nullable = true), `d`: STRING (nullable = true)> - Provider: parquet - Partition Columns: [`c`, `d`] - Storage(Location: sql/core/spark-warehouse/showdb.db/show_t1) - Partition Provider: Catalog) +Table: `showdb`.`show_t1` +Created: +Last Access: +Type: MANAGED +Provider: parquet +Partition Columns: [`c`, `d`] +Storage(Location: sql/core/spark-warehouse/showdb.db/show_t1) +Partition Provider: Catalog +Schema: root + |-- a: string (nullable = true) + |-- b: integer (nullable = true) + |-- c: string (nullable = true) + |-- d: string (nullable = true) +) showdb show_t2 false CatalogTable( - Table: `showdb`.`show_t2` - Created: - Last Access: - Type: MANAGED - Schema: STRUCT<`b`: STRING (nullable = true), `d`: INT (nullable = true)> - Provider: parquet - Storage(Location: sql/core/spark-warehouse/showdb.db/show_t2)) +Table: `showdb`.`show_t2` +Created: +Last Access: +Type: MANAGED +Provider: parquet +Storage(Location: sql/core/spark-warehouse/showdb.db/show_t2) +Schema: root + |-- b: string (nullable = true) + |-- d: integer (nullable = true) +) -- !query 13 From d9991479ad07a857e0af009abe579b7ddb283764 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Thu, 23 Mar 2017 00:06:43 -0700 Subject: [PATCH 03/15] revert --- .../main/scala/org/apache/spark/sql/types/StructType.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala index b4e28e302e5d..8d8b5b86d5aa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala @@ -343,9 +343,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru } override def sql: String = { - val fieldTypes = fields.map { f => - s"${quoteIdentifier(f.name)}: ${f.dataType.sql} (nullable = ${f.nullable})" - } + val fieldTypes = fields.map(f => s"${quoteIdentifier(f.name)}: ${f.dataType.sql}") s"STRUCT<${fieldTypes.mkString(", ")}>" } @@ -382,7 +380,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru override private[spark] def asNullable: StructType = { val newFields = fields.map { - case StructField(name, dataType, _, metadata) => + case StructField(name, dataType, nullable, metadata) => StructField(name, dataType.asNullable, nullable = true, metadata) } From 3859097269472d61171df349188b8b9ab03d5e8b Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Fri, 24 Mar 2017 16:14:35 -0700 Subject: [PATCH 04/15] address comments. --- .../sql/catalyst/catalog/interface.scala | 63 ++++++---- .../spark/sql/execution/command/tables.scala | 7 +- .../sql-tests/results/describe.sql.out | 117 +++++++++--------- .../sql-tests/results/show-tables.sql.out | 21 ++-- .../apache/spark/sql/SQLQueryTestSuite.scala | 6 +- 5 files changed, 113 insertions(+), 101 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 5d19702e300c..76ba70ab594d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -91,15 +91,20 @@ case class CatalogTablePartition( storage: CatalogStorageFormat, parameters: Map[String, String] = Map.empty) { - override def toString: String = { + private def toStringSeq: Seq[String] = { val specString = spec.map { case (k, v) => s"$k=$v" }.mkString(", ") - val output = - Seq( - s"Partition Values: [$specString]", - s"$storage", - s"Partition Parameters:{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}") + Seq( + s"Partition Values: [$specString]", + s"$storage", + s"Partition Parameters:{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}") + } + + override def toString: String = { + toStringSeq.filter(_.nonEmpty).mkString("CatalogPartition(\n\t", "\n\t", ")") + } - output.filter(_.nonEmpty).mkString("CatalogPartition(\n\t", "\n\t", ")") + def simpleString: String = { + toStringSeq.filter(_.nonEmpty).mkString("", "\n", "") } /** Return the partition location, assuming it is specified. */ @@ -261,7 +266,7 @@ case class CatalogTable( locationUri, inputFormat, outputFormat, serde, compressed, properties)) } - override def toString: String = { + private def toStringSeq: Seq[String] = { val tableProperties = properties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") val partitionColumns = partitionColumnNames.map(quoteIdentifier).mkString("[", ", ", "]") val bucketStrings = bucketSpec match { @@ -273,28 +278,32 @@ case class CatalogTable( if (bucketColumnNames.nonEmpty) s"Bucket Columns: $bucketColumnsString" else "", if (sortColumnNames.nonEmpty) s"Sort Columns: $sortColumnsString" else "" ) - case _ => Nil } - val output = - Seq(s"Table: ${identifier.quotedString}", - if (owner.nonEmpty) s"Owner: $owner" else "", - s"Created: ${new Date(createTime).toString}", - s"Last Access: ${new Date(lastAccessTime).toString}", - s"Type: ${tableType.name}", - if (provider.isDefined) s"Provider: ${provider.get}" else "", - if (partitionColumnNames.nonEmpty) s"Partition Columns: $partitionColumns" else "" - ) ++ bucketStrings ++ Seq( - viewText.map("View: " + _).getOrElse(""), - comment.map("Comment: " + _).getOrElse(""), - if (properties.nonEmpty) s"Properties: $tableProperties" else "", - if (stats.isDefined) s"Statistics: ${stats.get.simpleString}" else "", - s"$storage", - if (tracksPartitionsInCatalog) "Partition Provider: Catalog" else "", - if (schema.nonEmpty) s"Schema: ${schema.treeString}" else "") - - output.filter(_.nonEmpty).mkString("CatalogTable(\n", "\n", ")") + Seq(s"Table: ${identifier.quotedString}", + if (owner.nonEmpty) s"Owner: $owner" else "", + s"Created: ${new Date(createTime).toString}", + s"Last Access: ${new Date(lastAccessTime).toString}", + s"Type: ${tableType.name}", + if (provider.isDefined) s"Provider: ${provider.get}" else "", + if (partitionColumnNames.nonEmpty) s"Partition Columns: $partitionColumns" else "" + ) ++ bucketStrings ++ Seq( + viewText.map("View: " + _).getOrElse(""), + comment.map("Comment: " + _).getOrElse(""), + if (properties.nonEmpty) s"Properties: $tableProperties" else "", + if (stats.isDefined) s"Statistics: ${stats.get.simpleString}" else "", + s"$storage", + if (tracksPartitionsInCatalog) "Partition Provider: Catalog" else "", + if (schema.nonEmpty) s"Schema: ${schema.treeString}" else "") + } + + override def toString: String = { + toStringSeq.filter(_.nonEmpty).mkString("CatalogTable(\n", "\n", ")") + } + + def simpleString: String = { + toStringSeq.filter(_.nonEmpty).mkString("", "\n", "") } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index c7aeef06a0bf..30436358a5d2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -560,7 +560,8 @@ case class DescribeTableCommand( private def describeExtendedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") - append(buffer, "# Detailed Table Information", table.toString, "") + append(buffer, "# Detailed Table Information", "", "") + append(buffer, table.simpleString, "", "") } private def describeFormattedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { @@ -728,7 +729,7 @@ case class ShowTablesCommand( val tableName = tableIdent.table val isTemp = catalog.isTemporaryTable(tableIdent) if (isExtended) { - val information = catalog.getTempViewOrPermanentTableMetadata(tableIdent).toString + val information = catalog.getTempViewOrPermanentTableMetadata(tableIdent).simpleString Row(database, tableName, isTemp, s"$information\n") } else { Row(database, tableName, isTemp) @@ -745,7 +746,7 @@ case class ShowTablesCommand( val database = table.database.getOrElse("") val tableName = table.table val isTemp = catalog.isTemporaryTable(table) - val information = partition.toString + val information = partition.simpleString Seq(Row(database, tableName, isTemp, s"$information\n")) } } diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 1d9370f2869d..8d1beeae02cc 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -23,13 +23,13 @@ DESCRIBE t -- !query 2 schema struct -- !query 2 output -# Partition Information -# col_name data_type comment a string b int c string -c string d string +# Partition Information +# col_name data_type comment +c string d string @@ -38,13 +38,13 @@ DESC t -- !query 3 schema struct -- !query 3 output -# Partition Information -# col_name data_type comment a string b int c string -c string d string +# Partition Information +# col_name data_type comment +c string d string @@ -53,13 +53,13 @@ DESC TABLE t -- !query 4 schema struct -- !query 4 output -# Partition Information -# col_name data_type comment a string b int c string -c string d string +# Partition Information +# col_name data_type comment +c string d string @@ -68,27 +68,29 @@ DESC FORMATTED t -- !query 5 schema struct -- !query 5 output -# Detailed Table Information +a string +b int +c string +d string # Partition Information -# Storage Information # col_name data_type comment -Comment: table_comment -Compressed: No -Created: +c string +d string + +# Detailed Table Information Database: default +Owner: +Created: Last Access: Location: sql/core/spark-warehouse/t -Owner: -Partition Provider: Catalog -Storage Desc Parameters: -Table Parameters: Table Type: MANAGED -a string -b int -c string -c string -d string -d string +Comment: table_comment +Table Parameters: + +# Storage Information +Compressed: No +Storage Desc Parameters: +Partition Provider: Catalog -- !query 6 @@ -96,7 +98,16 @@ DESC EXTENDED t -- !query 6 schema struct -- !query 6 output -# Detailed Table Information CatalogTable( +a string +b int +c string +d string +# Partition Information +# col_name data_type comment +c string +d string + +# Detailed Table Information Table: `default`.`t` Created: Last Access: @@ -111,15 +122,6 @@ Schema: root |-- b: integer (nullable = true) |-- c: string (nullable = true) |-- d: string (nullable = true) -) -# Partition Information -# col_name data_type comment -a string -b int -c string -c string -d string -d string -- !query 7 @@ -127,13 +129,13 @@ DESC t PARTITION (c='Us', d=1) -- !query 7 schema struct -- !query 7 output -# Partition Information -# col_name data_type comment a string b int c string -c string d string +# Partition Information +# col_name data_type comment +c string d string @@ -142,18 +144,19 @@ DESC EXTENDED t PARTITION (c='Us', d=1) -- !query 8 schema struct -- !query 8 output -# Partition Information -# col_name data_type comment -Detailed Partition Information CatalogPartition( - Partition Values: [c=Us, d=1] - Storage(Location: sql/core/spark-warehouse/t/c=Us/d=1) - Partition Parameters:{}) a string b int c string +d string +# Partition Information +# col_name data_type comment c string d string -d string + +Detailed Partition Information CatalogPartition( + Partition Values: [c=Us, d=1] + Storage(Location: sql/core/spark-warehouse/t/c=Us/d=1) + Partition Parameters:{}) -- !query 9 @@ -161,23 +164,25 @@ DESC FORMATTED t PARTITION (c='Us', d=1) -- !query 9 schema struct -- !query 9 output -# Detailed Partition Information -# Partition Information -# Storage Information -# col_name data_type comment -Compressed: No -Database: default -Location: sql/core/spark-warehouse/t/c=Us/d=1 -Partition Parameters: -Partition Value: [Us, 1] -Storage Desc Parameters: -Table: t a string b int c string +d string +# Partition Information +# col_name data_type comment c string d string -d string + +# Detailed Partition Information +Partition Value: [Us, 1] +Database: default +Table: t +Location: sql/core/spark-warehouse/t/c=Us/d=1 +Partition Parameters: + +# Storage Information +Compressed: No +Storage Desc Parameters: -- !query 10 diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out index 9d52caedddb8..8d473d00bdd4 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out @@ -118,18 +118,16 @@ SHOW TABLE EXTENDED LIKE 'show_t*' -- !query 12 schema struct -- !query 12 output -show_t3 true CatalogTable( -Table: `show_t3` +show_t3 true Table: `show_t3` Created: Last Access: Type: VIEW Storage() Schema: root |-- e: integer (nullable = true) -) -showdb show_t1 false CatalogTable( -Table: `showdb`.`show_t1` + +showdb show_t1 false Table: `showdb`.`show_t1` Created: Last Access: Type: MANAGED @@ -142,10 +140,9 @@ Schema: root |-- b: integer (nullable = true) |-- c: string (nullable = true) |-- d: string (nullable = true) -) -showdb show_t2 false CatalogTable( -Table: `showdb`.`show_t2` + +showdb show_t2 false Table: `showdb`.`show_t2` Created: Last Access: Type: MANAGED @@ -154,7 +151,6 @@ Storage(Location: sql/core/spark-warehouse/showdb.db/show_t2) Schema: root |-- b: string (nullable = true) |-- d: integer (nullable = true) -) -- !query 13 @@ -176,10 +172,9 @@ SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us', d=1) -- !query 14 schema struct -- !query 14 output -showdb show_t1 false CatalogPartition( - Partition Values: [c=Us, d=1] - Storage(Location: sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1) - Partition Parameters:{}) +showdb show_t1 false Partition Values: [c=Us, d=1] +Storage(Location: sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1) +Partition Parameters:{} -- !query 15 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 4092862c430b..8a5aa96bf1ad 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile} +import org.apache.spark.sql.execution.command.DescribeTableCommand import org.apache.spark.sql.test.SharedSQLContext import org.apache.spark.sql.types.StructType @@ -165,8 +166,8 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { s"-- Number of queries: ${outputs.size}\n\n\n" + outputs.zipWithIndex.map{case (qr, i) => qr.toString(i)}.mkString("\n\n\n") + "\n" } - val resultFile = new File(testCase.resultFile); - val parent = resultFile.getParentFile(); + val resultFile = new File(testCase.resultFile) + val parent = resultFile.getParentFile if (!parent.exists()) { assert(parent.mkdirs(), "Could not create directory: " + parent) } @@ -214,6 +215,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { // Returns true if the plan is supposed to be sorted. def isSorted(plan: LogicalPlan): Boolean = plan match { case _: Join | _: Aggregate | _: Generate | _: Sample | _: Distinct => false + case _: DescribeTableCommand => true case PhysicalOperation(_, _, Sort(_, true, _)) => true case _ => plan.children.iterator.exists(isSorted) } From 1d720792b4bb2619bb9273b4bfb06f850b182896 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Fri, 24 Mar 2017 19:46:14 -0700 Subject: [PATCH 05/15] fixed the test case --- .../org/apache/spark/sql/hive/execution/HiveComparisonTest.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala index 536ca8fd9d45..313757706248 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala @@ -207,6 +207,7 @@ abstract class HiveComparisonTest // This list contains indicators for those lines which do not have actual results and we // want to ignore. lazy val ignoredLineIndicators = Seq( + "# Detailed Table Information", "# Partition Information", "# col_name" ) From 68bb05ca29756eac8067b8977c555632873b09b3 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Sun, 26 Mar 2017 00:02:24 -0700 Subject: [PATCH 06/15] fix. --- .../sql/catalyst/catalog/interface.scala | 119 +++++++++++------- .../spark/sql/execution/command/tables.scala | 77 ++---------- .../sql-tests/results/describe.sql.out | 40 +++--- .../sql-tests/results/show-tables.sql.out | 10 +- 4 files changed, 106 insertions(+), 140 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 76ba70ab594d..47cbca089110 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -21,6 +21,7 @@ import java.net.URI import java.util.Date import com.google.common.base.Objects +import scala.collection.mutable import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{CatalystConf, FunctionIdentifier, InternalRow, TableIdentifier} @@ -57,20 +58,25 @@ case class CatalogStorageFormat( properties: Map[String, String]) { override def toString: String = { - val serdePropsToString = CatalogUtils.maskCredentials(properties) match { - case props if props.isEmpty => "" - case props => "Properties: " + props.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") - } - val output = - Seq(locationUri.map("Location: " + _).getOrElse(""), - inputFormat.map("InputFormat: " + _).getOrElse(""), - outputFormat.map("OutputFormat: " + _).getOrElse(""), - if (compressed) "Compressed" else "", - serde.map("Serde: " + _).getOrElse(""), - serdePropsToString) - output.filter(_.nonEmpty).mkString("Storage(", ", ", ")") + toLinkedHashMap.map { case ((key, value)) => + if (value.isEmpty) key else s"$key: $value" + }.mkString("Storage(", ", ", ")") } + def toLinkedHashMap: mutable.LinkedHashMap[String, String] = { + val map = new mutable.LinkedHashMap[String, String]() + locationUri.foreach(l => map.put("Location", l.toString)) + serde.foreach(map.put("Serde Library", _)) + inputFormat.foreach(map.put("InputFormat", _)) + outputFormat.foreach(map.put("OutputFormat", _)) + if (compressed) map.put("Compressed", "") + CatalogUtils.maskCredentials(properties) match { + case props if props.isEmpty => // No-op + case props => + map.put("Properties", props.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")) + } + map + } } object CatalogStorageFormat { @@ -91,20 +97,27 @@ case class CatalogTablePartition( storage: CatalogStorageFormat, parameters: Map[String, String] = Map.empty) { - private def toStringSeq: Seq[String] = { + def toLinkedHashMap: mutable.LinkedHashMap[String, String] = { + val map = new mutable.LinkedHashMap[String, String]() val specString = spec.map { case (k, v) => s"$k=$v" }.mkString(", ") - Seq( - s"Partition Values: [$specString]", - s"$storage", - s"Partition Parameters:{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}") + map.put("Partition Values", s"[$specString]") + map ++= storage.toLinkedHashMap + if (parameters.nonEmpty) { + map.put("Partition Parameters", s"{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}") + } + map } override def toString: String = { - toStringSeq.filter(_.nonEmpty).mkString("CatalogPartition(\n\t", "\n\t", ")") + toLinkedHashMap.map { case ((key, value)) => + if (value.isEmpty) key else s"$key: $value" + }.mkString("CatalogPartition(\n\t", "\n\t", ")") } def simpleString: String = { - toStringSeq.filter(_.nonEmpty).mkString("", "\n", "") + toLinkedHashMap.map { case ((key, value)) => + if (value.isEmpty) key else s"$key: $value" + }.mkString("", "\n", "") } /** Return the partition location, assuming it is specified. */ @@ -159,6 +172,14 @@ case class BucketSpec( } s"$numBuckets buckets, $bucketString$sortString" } + + def toLinkedHashMap: mutable.LinkedHashMap[String, String] = { + mutable.LinkedHashMap[String, String]( + "Num Buckets" -> numBuckets.toString, + "Bucket Columns" -> bucketColumnNames.map(quoteIdentifier).mkString("[", ", ", "]"), + "Sort Columns" -> sortColumnNames.map(quoteIdentifier).mkString("[", ", ", "]") + ) + } } /** @@ -266,44 +287,48 @@ case class CatalogTable( locationUri, inputFormat, outputFormat, serde, compressed, properties)) } - private def toStringSeq: Seq[String] = { + + def toLinkedHashMap: mutable.LinkedHashMap[String, String] = { + val map = new mutable.LinkedHashMap[String, String]() val tableProperties = properties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") val partitionColumns = partitionColumnNames.map(quoteIdentifier).mkString("[", ", ", "]") - val bucketStrings = bucketSpec match { - case Some(BucketSpec(numBuckets, bucketColumnNames, sortColumnNames)) => - val bucketColumnsString = bucketColumnNames.map(quoteIdentifier).mkString("[", ", ", "]") - val sortColumnsString = sortColumnNames.map(quoteIdentifier).mkString("[", ", ", "]") - Seq( - s"Num Buckets: $numBuckets", - if (bucketColumnNames.nonEmpty) s"Bucket Columns: $bucketColumnsString" else "", - if (sortColumnNames.nonEmpty) s"Sort Columns: $sortColumnsString" else "" - ) - case _ => Nil + + map.put("Table", identifier.quotedString) + if (owner.nonEmpty) map.put("Owner", owner) + map.put("Created", new Date(createTime).toString) + map.put("Last Access", new Date(lastAccessTime).toString) + map.put("Type", tableType.name) + provider.foreach(map.put("Provider", _)) + bucketSpec.foreach(map ++= _.toLinkedHashMap) + comment.foreach(map.put("Comment", _)) + if (tableType == CatalogTableType.VIEW) { + viewText.foreach(map.put("View Text", _)) + viewDefaultDatabase.foreach(map.put("View Default Database", _)) + if (viewQueryColumnNames.nonEmpty) { + map.put("View Query Output Columns", viewQueryColumnNames.mkString("[", ", ", "]")) + } } - Seq(s"Table: ${identifier.quotedString}", - if (owner.nonEmpty) s"Owner: $owner" else "", - s"Created: ${new Date(createTime).toString}", - s"Last Access: ${new Date(lastAccessTime).toString}", - s"Type: ${tableType.name}", - if (provider.isDefined) s"Provider: ${provider.get}" else "", - if (partitionColumnNames.nonEmpty) s"Partition Columns: $partitionColumns" else "" - ) ++ bucketStrings ++ Seq( - viewText.map("View: " + _).getOrElse(""), - comment.map("Comment: " + _).getOrElse(""), - if (properties.nonEmpty) s"Properties: $tableProperties" else "", - if (stats.isDefined) s"Statistics: ${stats.get.simpleString}" else "", - s"$storage", - if (tracksPartitionsInCatalog) "Partition Provider: Catalog" else "", - if (schema.nonEmpty) s"Schema: ${schema.treeString}" else "") + if (properties.nonEmpty) map.put("Properties", tableProperties) + stats.foreach(s => map.put("Statistics", s.simpleString)) + map ++= storage.toLinkedHashMap + if (tracksPartitionsInCatalog) map.put("Partition Provider", "Catalog") + if (partitionColumnNames.nonEmpty) map.put("Partition Columns", partitionColumns) + if (schema.nonEmpty) map.put("Schema", schema.treeString) + + map } override def toString: String = { - toStringSeq.filter(_.nonEmpty).mkString("CatalogTable(\n", "\n", ")") + toLinkedHashMap.map { case ((key, value)) => + if (value.isEmpty) key else s"$key: $value" + }.mkString("CatalogTable(\n", "\n", ")") } def simpleString: String = { - toStringSeq.filter(_.nonEmpty).mkString("", "\n", "") + toLinkedHashMap.map { case ((key, value)) => + if (value.isEmpty) key else s"$key: $value" + }.mkString("", "\n", "") } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 30436358a5d2..cd350bb0cafc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -567,64 +567,7 @@ case class DescribeTableCommand( private def describeFormattedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") append(buffer, "# Detailed Table Information", "", "") - append(buffer, "Database:", table.database, "") - append(buffer, "Owner:", table.owner, "") - append(buffer, "Created:", new Date(table.createTime).toString, "") - append(buffer, "Last Access:", new Date(table.lastAccessTime).toString, "") - append(buffer, "Location:", table.storage.locationUri.map(CatalogUtils.URIToString(_)) - .getOrElse(""), "") - append(buffer, "Table Type:", table.tableType.name, "") - append(buffer, "Comment:", table.comment.getOrElse(""), "") - table.stats.foreach(s => append(buffer, "Statistics:", s.simpleString, "")) - - append(buffer, "Table Parameters:", "", "") - table.properties.foreach { case (key, value) => - append(buffer, s" $key", value, "") - } - - describeStorageInfo(table, buffer) - - if (table.tableType == CatalogTableType.VIEW) describeViewInfo(table, buffer) - - if (DDLUtils.isDatasourceTable(table) && table.tracksPartitionsInCatalog) { - append(buffer, "Partition Provider:", "Catalog", "") - } - } - - private def describeStorageInfo(metadata: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { - append(buffer, "", "", "") - append(buffer, "# Storage Information", "", "") - metadata.storage.serde.foreach(serdeLib => append(buffer, "SerDe Library:", serdeLib, "")) - metadata.storage.inputFormat.foreach(format => append(buffer, "InputFormat:", format, "")) - metadata.storage.outputFormat.foreach(format => append(buffer, "OutputFormat:", format, "")) - append(buffer, "Compressed:", if (metadata.storage.compressed) "Yes" else "No", "") - describeBucketingInfo(metadata, buffer) - - append(buffer, "Storage Desc Parameters:", "", "") - val maskedProperties = CatalogUtils.maskCredentials(metadata.storage.properties) - maskedProperties.foreach { case (key, value) => - append(buffer, s" $key", value, "") - } - } - - private def describeViewInfo(metadata: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { - append(buffer, "", "", "") - append(buffer, "# View Information", "", "") - append(buffer, "View Text:", metadata.viewText.getOrElse(""), "") - append(buffer, "View Default Database:", metadata.viewDefaultDatabase.getOrElse(""), "") - append(buffer, "View Query Output Columns:", - metadata.viewQueryColumnNames.mkString("[", ", ", "]"), "") - } - - private def describeBucketingInfo(metadata: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { - metadata.bucketSpec match { - case Some(BucketSpec(numBuckets, bucketColumnNames, sortColumnNames)) => - append(buffer, "Num Buckets:", numBuckets.toString, "") - append(buffer, "Bucket Columns:", bucketColumnNames.mkString("[", ", ", "]"), "") - append(buffer, "Sort Columns:", sortColumnNames.mkString("[", ", ", "]"), "") - - case _ => - } + table.toLinkedHashMap.foreach(s => append(buffer, s._1 + ":", s._2, "")) } private def describeDetailedPartitionInfo( @@ -642,7 +585,6 @@ case class DescribeTableCommand( describeExtendedDetailedPartitionInfo(table, metadata, partition, result) } else if (isFormatted) { describeFormattedDetailedPartitionInfo(table, metadata, partition, result) - describeStorageInfo(metadata, result) } } @@ -652,7 +594,8 @@ case class DescribeTableCommand( partition: CatalogTablePartition, buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") - append(buffer, "Detailed Partition Information " + partition.toString, "", "") + append(buffer, "# Detailed Partition Information", "", "") + append(buffer, partition.simpleString, "", "") } private def describeFormattedDetailedPartitionInfo( @@ -662,15 +605,17 @@ case class DescribeTableCommand( buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") append(buffer, "# Detailed Partition Information", "", "") - append(buffer, "Partition Value:", s"[${partition.spec.values.mkString(", ")}]", "") append(buffer, "Database:", table.database, "") append(buffer, "Table:", tableIdentifier.table, "") - append(buffer, "Location:", partition.storage.locationUri.map(CatalogUtils.URIToString(_)) - .getOrElse(""), "") - append(buffer, "Partition Parameters:", "", "") - partition.parameters.foreach { case (key, value) => - append(buffer, s" $key", value, "") + partition.toLinkedHashMap.foreach(s => append(buffer, s._1 + ":", s._2, "")) + append(buffer, "", "", "") + append(buffer, "# Table Storage Information", "", "") + table.bucketSpec match { + case Some(spec) => + spec.toLinkedHashMap.foreach(s => append(buffer, s._1 + ":", s._2, "")) + case _ => } + table.storage.toLinkedHashMap.foreach(s => append(buffer, s._1 + ":", s._2, "")) } private def describeSchema(schema: StructType, buffer: ArrayBuffer[Row]): Unit = { diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 8d1beeae02cc..aafc7c634ce8 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -78,19 +78,20 @@ c string d string # Detailed Table Information -Database: default -Owner: +Table: `default`.`t` Created: Last Access: -Location: sql/core/spark-warehouse/t -Table Type: MANAGED +Type: MANAGED +Provider: parquet Comment: table_comment -Table Parameters: - -# Storage Information -Compressed: No -Storage Desc Parameters: -Partition Provider: Catalog +Location: sql/core/spark-warehouse/t +Partition Provider: Catalog +Partition Columns: [`c`, `d`] +Schema: root + |-- a: string (nullable = true) + |-- b: integer (nullable = true) + |-- c: string (nullable = true) + |-- d: string (nullable = true) -- !query 6 @@ -113,10 +114,10 @@ Created: Last Access: Type: MANAGED Provider: parquet -Partition Columns: [`c`, `d`] Comment: table_comment -Storage(Location: sql/core/spark-warehouse/t) +Location: sql/core/spark-warehouse/t Partition Provider: Catalog +Partition Columns: [`c`, `d`] Schema: root |-- a: string (nullable = true) |-- b: integer (nullable = true) @@ -153,10 +154,9 @@ d string c string d string -Detailed Partition Information CatalogPartition( - Partition Values: [c=Us, d=1] - Storage(Location: sql/core/spark-warehouse/t/c=Us/d=1) - Partition Parameters:{}) +# Detailed Partition Information +Partition Values: [c=Us, d=1] +Location: sql/core/spark-warehouse/t/c=Us/d=1 -- !query 9 @@ -174,15 +174,13 @@ c string d string # Detailed Partition Information -Partition Value: [Us, 1] Database: default Table: t +Partition Values: [c=Us, d=1] Location: sql/core/spark-warehouse/t/c=Us/d=1 -Partition Parameters: -# Storage Information -Compressed: No -Storage Desc Parameters: +# Table Storage Information +Location: sql/core/spark-warehouse/t -- !query 10 diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out index 8d473d00bdd4..8a4bef05a59e 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out @@ -122,7 +122,6 @@ show_t3 true Table: `show_t3` Created: Last Access: Type: VIEW -Storage() Schema: root |-- e: integer (nullable = true) @@ -132,9 +131,9 @@ Created: Last Access: Type: MANAGED Provider: parquet -Partition Columns: [`c`, `d`] -Storage(Location: sql/core/spark-warehouse/showdb.db/show_t1) +Location: sql/core/spark-warehouse/showdb.db/show_t1 Partition Provider: Catalog +Partition Columns: [`c`, `d`] Schema: root |-- a: string (nullable = true) |-- b: integer (nullable = true) @@ -147,7 +146,7 @@ Created: Last Access: Type: MANAGED Provider: parquet -Storage(Location: sql/core/spark-warehouse/showdb.db/show_t2) +Location: sql/core/spark-warehouse/showdb.db/show_t2 Schema: root |-- b: string (nullable = true) |-- d: integer (nullable = true) @@ -173,8 +172,7 @@ SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us', d=1) struct -- !query 14 output showdb show_t1 false Partition Values: [c=Us, d=1] -Storage(Location: sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1) -Partition Parameters:{} +Location: sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1 -- !query 15 From ac3f35144f3d38479873beb47b3d53b03de61b82 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Sun, 26 Mar 2017 11:05:18 -0700 Subject: [PATCH 07/15] migrate the test case --- .../resources/sql-tests/inputs/describe.sql | 40 ++- .../sql-tests/results/describe.sql.out | 255 +++++++++++++++--- .../sql/hive/execution/HiveDDLSuite.scala | 91 ------- .../sql/hive/execution/SQLQuerySuite.scala | 73 ----- 4 files changed, 255 insertions(+), 204 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql index 56f3281440d2..3830616da82c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql @@ -1,4 +1,10 @@ -CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet PARTITIONED BY (c, d) COMMENT 'table_comment'; +CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet + PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS + COMMENT 'table_comment'; + +CREATE TEMPORARY VIEW temp_v AS SELECT * FROM t; + +CREATE VIEW v AS SELECT * FROM t; ALTER TABLE t ADD PARTITION (c='Us', d=1); @@ -27,5 +33,35 @@ DESC t PARTITION (c='Us'); -- ParseException: PARTITION specification is incomplete DESC t PARTITION (c='Us', d); --- DROP TEST TABLE +-- DESC Temp View + +DESC temp_v; + +DESC TABLE temp_v; + +DESC FORMATTED temp_v; + +DESC EXTENDED temp_v; + +-- AnalysisException DESC PARTITION is not allowed on a temporary view +DESC temp_v PARTITION (c='Us', d=1); + +-- DESC Persistent View + +DESC v; + +DESC TABLE v; + +DESC FORMATTED v; + +DESC EXTENDED v; + +-- AnalysisException DESC PARTITION is not allowed on a view +DESC v PARTITION (c='Us', d=1); + +-- DROP TEST TABLES/VIEWS DROP TABLE t; + +DROP VIEW temp_v; + +DROP VIEW v; diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index aafc7c634ce8..20613fc864ef 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -1,9 +1,11 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 14 +-- Number of queries: 28 -- !query 0 -CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet PARTITIONED BY (c, d) COMMENT 'table_comment' +CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet + PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS + COMMENT 'table_comment' -- !query 0 schema struct<> -- !query 0 output @@ -11,7 +13,7 @@ struct<> -- !query 1 -ALTER TABLE t ADD PARTITION (c='Us', d=1) +CREATE TEMPORARY VIEW temp_v AS SELECT * FROM t -- !query 1 schema struct<> -- !query 1 output @@ -19,10 +21,26 @@ struct<> -- !query 2 -DESCRIBE t +CREATE VIEW v AS SELECT * FROM t -- !query 2 schema -struct +struct<> -- !query 2 output + + + +-- !query 3 +ALTER TABLE t ADD PARTITION (c='Us', d=1) +-- !query 3 schema +struct<> +-- !query 3 output + + + +-- !query 4 +DESCRIBE t +-- !query 4 schema +struct +-- !query 4 output a string b int c string @@ -33,11 +51,11 @@ c string d string --- !query 3 +-- !query 5 DESC t --- !query 3 schema +-- !query 5 schema struct --- !query 3 output +-- !query 5 output a string b int c string @@ -48,11 +66,11 @@ c string d string --- !query 4 +-- !query 6 DESC TABLE t --- !query 4 schema +-- !query 6 schema struct --- !query 4 output +-- !query 6 output a string b int c string @@ -63,11 +81,11 @@ c string d string --- !query 5 +-- !query 7 DESC FORMATTED t --- !query 5 schema +-- !query 7 schema struct --- !query 5 output +-- !query 7 output a string b int c string @@ -83,6 +101,9 @@ Created: Last Access: Type: MANAGED Provider: parquet +Num Buckets: 2 +Bucket Columns: [`a`] +Sort Columns: [`b`] Comment: table_comment Location: sql/core/spark-warehouse/t Partition Provider: Catalog @@ -94,11 +115,11 @@ Schema: root |-- d: string (nullable = true) --- !query 6 +-- !query 8 DESC EXTENDED t --- !query 6 schema +-- !query 8 schema struct --- !query 6 output +-- !query 8 output a string b int c string @@ -114,6 +135,9 @@ Created: Last Access: Type: MANAGED Provider: parquet +Num Buckets: 2 +Bucket Columns: [`a`] +Sort Columns: [`b`] Comment: table_comment Location: sql/core/spark-warehouse/t Partition Provider: Catalog @@ -125,11 +149,11 @@ Schema: root |-- d: string (nullable = true) --- !query 7 +-- !query 9 DESC t PARTITION (c='Us', d=1) --- !query 7 schema +-- !query 9 schema struct --- !query 7 output +-- !query 9 output a string b int c string @@ -140,11 +164,11 @@ c string d string --- !query 8 +-- !query 10 DESC EXTENDED t PARTITION (c='Us', d=1) --- !query 8 schema +-- !query 10 schema struct --- !query 8 output +-- !query 10 output a string b int c string @@ -159,11 +183,11 @@ Partition Values: [c=Us, d=1] Location: sql/core/spark-warehouse/t/c=Us/d=1 --- !query 9 +-- !query 11 DESC FORMATTED t PARTITION (c='Us', d=1) --- !query 9 schema +-- !query 11 schema struct --- !query 9 output +-- !query 11 output a string b int c string @@ -180,34 +204,37 @@ Partition Values: [c=Us, d=1] Location: sql/core/spark-warehouse/t/c=Us/d=1 # Table Storage Information +Num Buckets: 2 +Bucket Columns: [`a`] +Sort Columns: [`b`] Location: sql/core/spark-warehouse/t --- !query 10 +-- !query 12 DESC t PARTITION (c='Us', d=2) --- !query 10 schema +-- !query 12 schema struct<> --- !query 10 output +-- !query 12 output org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException Partition not found in table 't' database 'default': c -> Us d -> 2; --- !query 11 +-- !query 13 DESC t PARTITION (c='Us') --- !query 11 schema +-- !query 13 schema struct<> --- !query 11 output +-- !query 13 output org.apache.spark.sql.AnalysisException Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`'; --- !query 12 +-- !query 14 DESC t PARTITION (c='Us', d) --- !query 12 schema +-- !query 14 schema struct<> --- !query 12 output +-- !query 14 output org.apache.spark.sql.catalyst.parser.ParseException PARTITION specification is incomplete: `d`(line 1, pos 0) @@ -217,9 +244,161 @@ DESC t PARTITION (c='Us', d) ^^^ --- !query 13 +-- !query 15 +DESC temp_v +-- !query 15 schema +struct +-- !query 15 output +a string +b int +c string +d string + + +-- !query 16 +DESC TABLE temp_v +-- !query 16 schema +struct +-- !query 16 output +a string +b int +c string +d string + + +-- !query 17 +DESC FORMATTED temp_v +-- !query 17 schema +struct +-- !query 17 output +a string +b int +c string +d string + + +-- !query 18 +DESC EXTENDED temp_v +-- !query 18 schema +struct +-- !query 18 output +a string +b int +c string +d string + + +-- !query 19 +DESC temp_v PARTITION (c='Us', d=1) +-- !query 19 schema +struct<> +-- !query 19 output +org.apache.spark.sql.AnalysisException +DESC PARTITION is not allowed on a temporary view: temp_v; + + +-- !query 20 +DESC v +-- !query 20 schema +struct +-- !query 20 output +a string +b int +c string +d string + + +-- !query 21 +DESC TABLE v +-- !query 21 schema +struct +-- !query 21 output +a string +b int +c string +d string + + +-- !query 22 +DESC FORMATTED v +-- !query 22 schema +struct +-- !query 22 output +a string +b int +c string +d string + +# Detailed Table Information +Table: `default`.`v` +Created: +Last Access: +Type: VIEW +View Text: SELECT * FROM t +View Default Database: default +View Query Output Columns: [a, b, c, d] +Properties: [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] +Schema: root + |-- a: string (nullable = true) + |-- b: integer (nullable = true) + |-- c: string (nullable = true) + |-- d: string (nullable = true) + + +-- !query 23 +DESC EXTENDED v +-- !query 23 schema +struct +-- !query 23 output +a string +b int +c string +d string + +# Detailed Table Information +Table: `default`.`v` +Created: +Last Access: +Type: VIEW +View Text: SELECT * FROM t +View Default Database: default +View Query Output Columns: [a, b, c, d] +Properties: [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] +Schema: root + |-- a: string (nullable = true) + |-- b: integer (nullable = true) + |-- c: string (nullable = true) + |-- d: string (nullable = true) + + +-- !query 24 +DESC v PARTITION (c='Us', d=1) +-- !query 24 schema +struct<> +-- !query 24 output +org.apache.spark.sql.AnalysisException +DESC PARTITION is not allowed on a view: v; + + +-- !query 25 DROP TABLE t --- !query 13 schema +-- !query 25 schema struct<> --- !query 13 output +-- !query 25 output + + + +-- !query 26 +DROP VIEW temp_v +-- !query 26 schema +struct<> +-- !query 26 output + + + +-- !query 27 +DROP VIEW v +-- !query 27 schema +struct<> +-- !query 27 output diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 04bc79d43032..1f9a35612c2d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -708,23 +708,6 @@ class HiveDDLSuite } } - test("desc table for Hive table") { - withTable("tab1") { - val tabName = "tab1" - sql(s"CREATE TABLE $tabName(c1 int)") - - assert(sql(s"DESC $tabName").collect().length == 1) - - assert( - sql(s"DESC FORMATTED $tabName").collect() - .exists(_.getString(0) == "# Storage Information")) - - assert( - sql(s"DESC EXTENDED $tabName").collect() - .exists(_.getString(0) == "# Detailed Table Information")) - } - } - test("desc table for Hive table - partitioned table") { withTable("tbl") { sql("CREATE TABLE tbl(a int) PARTITIONED BY (b int)") @@ -741,23 +724,6 @@ class HiveDDLSuite } } - test("desc formatted table for permanent view") { - withTable("tbl") { - withView("view1") { - sql("CREATE TABLE tbl(a int)") - sql("CREATE VIEW view1 AS SELECT * FROM tbl") - assert(sql("DESC FORMATTED view1").collect().containsSlice( - Seq( - Row("# View Information", "", ""), - Row("View Text:", "SELECT * FROM tbl", ""), - Row("View Default Database:", "default", ""), - Row("View Query Output Columns:", "[a]", "") - ) - )) - } - } - } - test("desc table for data source table using Hive Metastore") { assume(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive") val tabName = "tab1" @@ -1218,23 +1184,6 @@ class HiveDDLSuite sql(s"SELECT * FROM ${targetTable.identifier}")) } - test("desc table for data source table") { - withTable("tab1") { - val tabName = "tab1" - spark.range(1).write.format("json").saveAsTable(tabName) - - assert(sql(s"DESC $tabName").collect().length == 1) - - assert( - sql(s"DESC FORMATTED $tabName").collect() - .exists(_.getString(0) == "# Storage Information")) - - assert( - sql(s"DESC EXTENDED $tabName").collect() - .exists(_.getString(0) == "# Detailed Table Information")) - } - } - test("create table with the same name as an index table") { val tabName = "tab1" val indexName = tabName + "_index" @@ -1320,46 +1269,6 @@ class HiveDDLSuite } } - test("desc table for data source table - partitioned bucketed table") { - withTable("t1") { - spark - .range(1).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd).write - .bucketBy(2, "b").sortBy("c").partitionBy("d") - .saveAsTable("t1") - - val formattedDesc = sql("DESC FORMATTED t1").collect() - - assert(formattedDesc.containsSlice( - Seq( - Row("a", "bigint", null), - Row("b", "bigint", null), - Row("c", "bigint", null), - Row("d", "bigint", null), - Row("# Partition Information", "", ""), - Row("# col_name", "data_type", "comment"), - Row("d", "bigint", null), - Row("", "", ""), - Row("# Detailed Table Information", "", ""), - Row("Database:", "default", "") - ) - )) - - assert(formattedDesc.containsSlice( - Seq( - Row("Table Type:", "MANAGED", "") - ) - )) - - assert(formattedDesc.containsSlice( - Seq( - Row("Num Buckets:", "2", ""), - Row("Bucket Columns:", "[b]", ""), - Row("Sort Columns:", "[c]", "") - ) - )) - } - } - test("datasource and statistics table property keys are not allowed") { import org.apache.spark.sql.hive.HiveExternalCatalog.DATASOURCE_PREFIX import org.apache.spark.sql.hive.HiveExternalCatalog.STATISTICS_PREFIX diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 55ff4bb115e5..9743b337971a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -363,79 +363,6 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } - test("describe partition") { - withTable("partitioned_table") { - sql("CREATE TABLE partitioned_table (a STRING, b INT) PARTITIONED BY (c STRING, d STRING)") - sql("ALTER TABLE partitioned_table ADD PARTITION (c='Us', d=1)") - - checkKeywordsExist(sql("DESC partitioned_table PARTITION (c='Us', d=1)"), - "# Partition Information", - "# col_name") - - checkKeywordsExist(sql("DESC EXTENDED partitioned_table PARTITION (c='Us', d=1)"), - "# Partition Information", - "# col_name", - "Detailed Partition Information CatalogPartition(", - "Partition Values: [c=Us, d=1]", - "Storage(Location:", - "Partition Parameters") - - checkKeywordsExist(sql("DESC FORMATTED partitioned_table PARTITION (c='Us', d=1)"), - "# Partition Information", - "# col_name", - "# Detailed Partition Information", - "Partition Value:", - "Database:", - "Table:", - "Location:", - "Partition Parameters:", - "# Storage Information") - } - } - - test("describe partition - error handling") { - withTable("partitioned_table", "datasource_table") { - sql("CREATE TABLE partitioned_table (a STRING, b INT) PARTITIONED BY (c STRING, d STRING)") - sql("ALTER TABLE partitioned_table ADD PARTITION (c='Us', d=1)") - - val m = intercept[NoSuchPartitionException] { - sql("DESC partitioned_table PARTITION (c='Us', d=2)") - }.getMessage() - assert(m.contains("Partition not found in table")) - - val m2 = intercept[AnalysisException] { - sql("DESC partitioned_table PARTITION (c='Us')") - }.getMessage() - assert(m2.contains("Partition spec is invalid")) - - val m3 = intercept[ParseException] { - sql("DESC partitioned_table PARTITION (c='Us', d)") - }.getMessage() - assert(m3.contains("PARTITION specification is incomplete: `d`")) - - spark - .range(1).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd).write - .partitionBy("d") - .saveAsTable("datasource_table") - - sql("DESC datasource_table PARTITION (d=0)") - - val m5 = intercept[AnalysisException] { - spark.range(10).select('id as 'a, 'id as 'b).createTempView("view1") - sql("DESC view1 PARTITION (c='Us', d=1)") - }.getMessage() - assert(m5.contains("DESC PARTITION is not allowed on a temporary view")) - - withView("permanent_view") { - val m = intercept[AnalysisException] { - sql("CREATE VIEW permanent_view AS SELECT * FROM partitioned_table") - sql("DESC permanent_view PARTITION (c='Us', d=1)") - }.getMessage() - assert(m.contains("DESC PARTITION is not allowed on a view")) - } - } - } - test("SPARK-5371: union with null and sum") { val df = Seq((1, 1)).toDF("c1", "c2") df.createOrReplaceTempView("table1") From bef1134ef1c0e90848550f6bcdf85b6e05e3eccf Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Mon, 27 Mar 2017 21:26:13 -0700 Subject: [PATCH 08/15] remove : --- .../scala/org/apache/spark/sql/execution/command/tables.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index cd350bb0cafc..7191fb4b785c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -567,7 +567,7 @@ case class DescribeTableCommand( private def describeFormattedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") append(buffer, "# Detailed Table Information", "", "") - table.toLinkedHashMap.foreach(s => append(buffer, s._1 + ":", s._2, "")) + table.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, "")) } private def describeDetailedPartitionInfo( From 29817ea3861f142f4357ff3d80fb27ed24d77cf6 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Mon, 27 Mar 2017 21:52:45 -0700 Subject: [PATCH 09/15] test case fix. --- .../sql-tests/results/describe.sql.out | 60 +++++++++---------- .../sql-tests/results/show-tables.sql.out | 18 +++--- .../apache/spark/sql/SQLQueryTestSuite.scala | 6 +- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 20613fc864ef..44d55222167d 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -96,19 +96,19 @@ c string d string # Detailed Table Information -Table: `default`.`t` -Created: -Last Access: -Type: MANAGED -Provider: parquet -Num Buckets: 2 -Bucket Columns: [`a`] -Sort Columns: [`b`] -Comment: table_comment -Location: sql/core/spark-warehouse/t -Partition Provider: Catalog -Partition Columns: [`c`, `d`] -Schema: root +Table `default`.`t` +Created[removed by SQLQueryTestSuite] +Last Access[removed by SQLQueryTestSuite] +Type MANAGED +Provider parquet +Num Buckets 2 +Bucket Columns [`a`] +Sort Columns [`b`] +Comment table_comment +Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t +Partition Provider Catalog +Partition Columns [`c`, `d`] +Schema root |-- a: string (nullable = true) |-- b: integer (nullable = true) |-- c: string (nullable = true) @@ -131,15 +131,15 @@ d string # Detailed Table Information Table: `default`.`t` -Created: -Last Access: +Created[removed by SQLQueryTestSuite] +Last Access[removed by SQLQueryTestSuite] Type: MANAGED Provider: parquet Num Buckets: 2 Bucket Columns: [`a`] Sort Columns: [`b`] Comment: table_comment -Location: sql/core/spark-warehouse/t +Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t Partition Provider: Catalog Partition Columns: [`c`, `d`] Schema: root @@ -180,7 +180,7 @@ d string # Detailed Partition Information Partition Values: [c=Us, d=1] -Location: sql/core/spark-warehouse/t/c=Us/d=1 +Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t/c=Us/d=1 -- !query 11 @@ -201,13 +201,13 @@ d string Database: default Table: t Partition Values: [c=Us, d=1] -Location: sql/core/spark-warehouse/t/c=Us/d=1 +Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t/c=Us/d=1 # Table Storage Information Num Buckets: 2 Bucket Columns: [`a`] Sort Columns: [`b`] -Location: sql/core/spark-warehouse/t +Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t -- !query 12 @@ -330,15 +330,15 @@ c string d string # Detailed Table Information -Table: `default`.`v` -Created: -Last Access: -Type: VIEW -View Text: SELECT * FROM t -View Default Database: default -View Query Output Columns: [a, b, c, d] -Properties: [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] -Schema: root +Table `default`.`v` +Created[removed by SQLQueryTestSuite] +Last Access[removed by SQLQueryTestSuite] +Type VIEW +View Text SELECT * FROM t +View Default Database default +View Query Output Columns [a, b, c, d] +Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] +Schema root |-- a: string (nullable = true) |-- b: integer (nullable = true) |-- c: string (nullable = true) @@ -357,8 +357,8 @@ d string # Detailed Table Information Table: `default`.`v` -Created: -Last Access: +Created[removed by SQLQueryTestSuite] +Last Access[removed by SQLQueryTestSuite] Type: VIEW View Text: SELECT * FROM t View Default Database: default diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out index 8a4bef05a59e..a5e2cd538339 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out @@ -119,19 +119,19 @@ SHOW TABLE EXTENDED LIKE 'show_t*' struct -- !query 12 output show_t3 true Table: `show_t3` -Created: -Last Access: +Created[removed by SQLQueryTestSuite] +Last Access[removed by SQLQueryTestSuite] Type: VIEW Schema: root |-- e: integer (nullable = true) showdb show_t1 false Table: `showdb`.`show_t1` -Created: -Last Access: +Created[removed by SQLQueryTestSuite] +Last Access[removed by SQLQueryTestSuite] Type: MANAGED Provider: parquet -Location: sql/core/spark-warehouse/showdb.db/show_t1 +Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t1 Partition Provider: Catalog Partition Columns: [`c`, `d`] Schema: root @@ -142,11 +142,11 @@ Schema: root showdb show_t2 false Table: `showdb`.`show_t2` -Created: -Last Access: +Created[removed by SQLQueryTestSuite] +Last Access[removed by SQLQueryTestSuite] Type: MANAGED Provider: parquet -Location: sql/core/spark-warehouse/showdb.db/show_t2 +Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t2 Schema: root |-- b: string (nullable = true) |-- d: integer (nullable = true) @@ -172,7 +172,7 @@ SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us', d=1) struct -- !query 14 output showdb show_t1 false Partition Values: [c=Us, d=1] -Location: sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1 +Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1 -- !query 15 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 8a5aa96bf1ad..edc17e79f5c7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -225,9 +225,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { val schema = df.schema // Get answer, but also get rid of the #1234 expression ids that show up in explain plans val answer = df.queryExecution.hiveResultString().map(_.replaceAll("#\\d+", "#x") - .replaceAll("Location:.*/sql/core/", "Location: sql/core/") - .replaceAll("Created: .*", "Created: ") - .replaceAll("Last Access: .*", "Last Access: ")) + .replaceAll("Location.*/sql/core/", "Location:[removed by SQLQueryTestSuite]sql/core/") + .replaceAll("Created.*", "Created[removed by SQLQueryTestSuite]") + .replaceAll("Last Access.*", "Last Access[removed by SQLQueryTestSuite]")) // If the output is not pre-sorted, sort it. if (isSorted(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted) From 6c560415b086ca28ae4b90cabbc585f8f5f2814c Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Tue, 28 Mar 2017 12:17:15 -0700 Subject: [PATCH 10/15] address comments. --- .../spark/sql/execution/SparkSqlParser.scala | 3 +- .../spark/sql/execution/command/tables.scala | 43 +++--------- .../sql-tests/results/describe.sql.out | 66 +++++++++++-------- .../sql-tests/results/show-tables.sql.out | 6 +- .../apache/spark/sql/SQLQueryTestSuite.scala | 2 +- .../sql/execution/SparkSqlParserSuite.scala | 6 +- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 4 -- .../sql/hive/execution/SQLQuerySuite.scala | 58 +++++++--------- 8 files changed, 78 insertions(+), 110 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index d4f23f9dd518..80afb59b3e88 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -322,8 +322,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { DescribeTableCommand( visitTableIdentifier(ctx.tableIdentifier), partitionSpec, - ctx.EXTENDED != null, - ctx.FORMATTED != null) + ctx.EXTENDED != null || ctx.FORMATTED != null) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 7191fb4b785c..b94a7bb151c4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -500,7 +500,6 @@ case class TruncateTableCommand( case class DescribeTableCommand( table: TableIdentifier, partitionSpec: TablePartitionSpec, - isExtended: Boolean, isFormatted: Boolean) extends RunnableCommand { @@ -536,14 +535,10 @@ case class DescribeTableCommand( describePartitionInfo(metadata, result) - if (partitionSpec.isEmpty) { - if (isExtended) { - describeExtendedTableInfo(metadata, result) - } else if (isFormatted) { - describeFormattedTableInfo(metadata, result) - } - } else { + if (partitionSpec.nonEmpty) { describeDetailedPartitionInfo(sparkSession, catalog, metadata, result) + } else if (isFormatted) { + describeFormattedTableInfo(metadata, result) } } @@ -558,12 +553,6 @@ case class DescribeTableCommand( } } - private def describeExtendedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { - append(buffer, "", "", "") - append(buffer, "# Detailed Table Information", "", "") - append(buffer, table.simpleString, "", "") - } - private def describeFormattedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") append(buffer, "# Detailed Table Information", "", "") @@ -581,21 +570,7 @@ case class DescribeTableCommand( } DDLUtils.verifyPartitionProviderIsHive(spark, metadata, "DESC PARTITION") val partition = catalog.getPartition(table, partitionSpec) - if (isExtended) { - describeExtendedDetailedPartitionInfo(table, metadata, partition, result) - } else if (isFormatted) { - describeFormattedDetailedPartitionInfo(table, metadata, partition, result) - } - } - - private def describeExtendedDetailedPartitionInfo( - tableIdentifier: TableIdentifier, - table: CatalogTable, - partition: CatalogTablePartition, - buffer: ArrayBuffer[Row]): Unit = { - append(buffer, "", "", "") - append(buffer, "# Detailed Partition Information", "", "") - append(buffer, partition.simpleString, "", "") + if (isFormatted) describeFormattedDetailedPartitionInfo(table, metadata, partition, result) } private def describeFormattedDetailedPartitionInfo( @@ -605,17 +580,17 @@ case class DescribeTableCommand( buffer: ArrayBuffer[Row]): Unit = { append(buffer, "", "", "") append(buffer, "# Detailed Partition Information", "", "") - append(buffer, "Database:", table.database, "") - append(buffer, "Table:", tableIdentifier.table, "") - partition.toLinkedHashMap.foreach(s => append(buffer, s._1 + ":", s._2, "")) + append(buffer, "Database", table.database, "") + append(buffer, "Table", tableIdentifier.table, "") + partition.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, "")) append(buffer, "", "", "") append(buffer, "# Table Storage Information", "", "") table.bucketSpec match { case Some(spec) => - spec.toLinkedHashMap.foreach(s => append(buffer, s._1 + ":", s._2, "")) + spec.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, "")) case _ => } - table.storage.toLinkedHashMap.foreach(s => append(buffer, s._1 + ":", s._2, "")) + table.storage.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, "")) } private def describeSchema(schema: StructType, buffer: ArrayBuffer[Row]): Unit = { diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 44d55222167d..c7f4738f1ca7 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -105,7 +105,7 @@ Num Buckets 2 Bucket Columns [`a`] Sort Columns [`b`] Comment table_comment -Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t +Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t Partition Provider Catalog Partition Columns [`c`, `d`] Schema root @@ -130,19 +130,19 @@ c string d string # Detailed Table Information -Table: `default`.`t` +Table `default`.`t` Created[removed by SQLQueryTestSuite] Last Access[removed by SQLQueryTestSuite] -Type: MANAGED -Provider: parquet -Num Buckets: 2 -Bucket Columns: [`a`] -Sort Columns: [`b`] -Comment: table_comment -Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t -Partition Provider: Catalog -Partition Columns: [`c`, `d`] -Schema: root +Type MANAGED +Provider parquet +Num Buckets 2 +Bucket Columns [`a`] +Sort Columns [`b`] +Comment table_comment +Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t +Partition Provider Catalog +Partition Columns [`c`, `d`] +Schema root |-- a: string (nullable = true) |-- b: integer (nullable = true) |-- c: string (nullable = true) @@ -179,8 +179,16 @@ c string d string # Detailed Partition Information -Partition Values: [c=Us, d=1] -Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t/c=Us/d=1 +Database default +Table t +Partition Values [c=Us, d=1] +Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t/c=Us/d=1 + +# Table Storage Information +Num Buckets 2 +Bucket Columns [`a`] +Sort Columns [`b`] +Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t -- !query 11 @@ -198,16 +206,16 @@ c string d string # Detailed Partition Information -Database: default -Table: t -Partition Values: [c=Us, d=1] -Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t/c=Us/d=1 +Database default +Table t +Partition Values [c=Us, d=1] +Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t/c=Us/d=1 # Table Storage Information -Num Buckets: 2 -Bucket Columns: [`a`] -Sort Columns: [`b`] -Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t +Num Buckets 2 +Bucket Columns [`a`] +Sort Columns [`b`] +Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t -- !query 12 @@ -356,15 +364,15 @@ c string d string # Detailed Table Information -Table: `default`.`v` +Table `default`.`v` Created[removed by SQLQueryTestSuite] Last Access[removed by SQLQueryTestSuite] -Type: VIEW -View Text: SELECT * FROM t -View Default Database: default -View Query Output Columns: [a, b, c, d] -Properties: [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] -Schema: root +Type VIEW +View Text SELECT * FROM t +View Default Database default +View Query Output Columns [a, b, c, d] +Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] +Schema root |-- a: string (nullable = true) |-- b: integer (nullable = true) |-- c: string (nullable = true) diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out index a5e2cd538339..10923a25c971 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out @@ -131,7 +131,7 @@ Created[removed by SQLQueryTestSuite] Last Access[removed by SQLQueryTestSuite] Type: MANAGED Provider: parquet -Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t1 +Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t1 Partition Provider: Catalog Partition Columns: [`c`, `d`] Schema: root @@ -146,7 +146,7 @@ Created[removed by SQLQueryTestSuite] Last Access[removed by SQLQueryTestSuite] Type: MANAGED Provider: parquet -Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t2 +Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t2 Schema: root |-- b: string (nullable = true) |-- d: integer (nullable = true) @@ -172,7 +172,7 @@ SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us', d=1) struct -- !query 14 output showdb show_t1 false Partition Values: [c=Us, d=1] -Location:[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1 +Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1 -- !query 15 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index edc17e79f5c7..d4474efe6608 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -225,7 +225,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { val schema = df.schema // Get answer, but also get rid of the #1234 expression ids that show up in explain plans val answer = df.queryExecution.hiveResultString().map(_.replaceAll("#\\d+", "#x") - .replaceAll("Location.*/sql/core/", "Location:[removed by SQLQueryTestSuite]sql/core/") + .replaceAll("Location.*/sql/core/", "Location[removed by SQLQueryTestSuite]sql/core/") .replaceAll("Created.*", "Created[removed by SQLQueryTestSuite]") .replaceAll("Last Access.*", "Last Access[removed by SQLQueryTestSuite]")) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala index a4d012cd7611..6aaeecadead6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala @@ -224,13 +224,13 @@ class SparkSqlParserSuite extends PlanTest { test("SPARK-17328 Fix NPE with EXPLAIN DESCRIBE TABLE") { assertEqual("describe table t", DescribeTableCommand( - TableIdentifier("t"), Map.empty, isExtended = false, isFormatted = false)) + TableIdentifier("t"), Map.empty, isFormatted = false)) assertEqual("describe table extended t", DescribeTableCommand( - TableIdentifier("t"), Map.empty, isExtended = true, isFormatted = false)) + TableIdentifier("t"), Map.empty, isFormatted = true)) assertEqual("describe table formatted t", DescribeTableCommand( - TableIdentifier("t"), Map.empty, isExtended = false, isFormatted = true)) + TableIdentifier("t"), Map.empty, isFormatted = true)) intercept("explain describe tables x", "Unsupported SQL statement") } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 5463728ca0c1..f08be82ab662 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -806,10 +806,6 @@ class JDBCSuite extends SparkFunSuite sql(s"DESC FORMATTED $tableName").collect().foreach { r => assert(!r.toString().contains(password)) } - - sql(s"DESC EXTENDED $tableName").collect().foreach { r => - assert(!r.toString().contains(password)) - } } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 9743b337971a..d012797e1992 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -603,7 +603,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } test("CTAS with serde") { - sql("CREATE TABLE ctas1 AS SELECT key k, value FROM src ORDER BY k, value").collect() + sql("CREATE TABLE ctas1 AS SELECT key k, value FROM src ORDER BY k, value") sql( """CREATE TABLE ctas2 | ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" @@ -613,86 +613,76 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { | AS | SELECT key, value | FROM src - | ORDER BY key, value""".stripMargin).collect() + | ORDER BY key, value""".stripMargin) + + val storageCtas2 = spark.sessionState.catalog.getTableMetadata(TableIdentifier("ctas2")).storage + assert(storageCtas2.inputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileInputFormat")) + assert(storageCtas2.outputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileOutputFormat")) + assert(storageCtas2.serde == Some("org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe")) + sql( """CREATE TABLE ctas3 | ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\012' | STORED AS textfile AS | SELECT key, value | FROM src - | ORDER BY key, value""".stripMargin).collect() + | ORDER BY key, value""".stripMargin) // the table schema may like (key: integer, value: string) sql( """CREATE TABLE IF NOT EXISTS ctas4 AS - | SELECT 1 AS key, value FROM src LIMIT 1""".stripMargin).collect() + | SELECT 1 AS key, value FROM src LIMIT 1""".stripMargin) // do nothing cause the table ctas4 already existed. sql( """CREATE TABLE IF NOT EXISTS ctas4 AS - | SELECT key, value FROM src ORDER BY key, value""".stripMargin).collect() + | SELECT key, value FROM src ORDER BY key, value""".stripMargin) checkAnswer( sql("SELECT k, value FROM ctas1 ORDER BY k, value"), - sql("SELECT key, value FROM src ORDER BY key, value").collect().toSeq) + sql("SELECT key, value FROM src ORDER BY key, value")) checkAnswer( sql("SELECT key, value FROM ctas2 ORDER BY key, value"), sql( """ SELECT key, value FROM src - ORDER BY key, value""").collect().toSeq) + ORDER BY key, value""")) checkAnswer( sql("SELECT key, value FROM ctas3 ORDER BY key, value"), sql( """ SELECT key, value FROM src - ORDER BY key, value""").collect().toSeq) + ORDER BY key, value""")) intercept[AnalysisException] { sql( """CREATE TABLE ctas4 AS - | SELECT key, value FROM src ORDER BY key, value""".stripMargin).collect() + | SELECT key, value FROM src ORDER BY key, value""".stripMargin) } checkAnswer( sql("SELECT key, value FROM ctas4 ORDER BY key, value"), sql("SELECT key, value FROM ctas4 LIMIT 1").collect().toSeq) - /* - Disabled because our describe table does not output the serde information right now. - checkKeywordsExist(sql("DESC EXTENDED ctas2"), - "name:key", "type:string", "name:value", "ctas2", - "org.apache.hadoop.hive.ql.io.RCFileInputFormat", - "org.apache.hadoop.hive.ql.io.RCFileOutputFormat", - "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe", - "serde_p1=p1", "serde_p2=p2", "tbl_p1=p11", "tbl_p2=p22", "MANAGED_TABLE" - ) - */ - sql( """CREATE TABLE ctas5 | STORED AS parquet AS | SELECT key, value | FROM src - | ORDER BY key, value""".stripMargin).collect() + | ORDER BY key, value""".stripMargin) + val storageCtas5 = spark.sessionState.catalog.getTableMetadata(TableIdentifier("ctas5")).storage + assert(storageCtas5.inputFormat == + Some("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat")) + assert(storageCtas5.outputFormat == + Some("org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat")) + assert(storageCtas5.serde == + Some("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe")) - /* - Disabled because our describe table does not output the serde information right now. - withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") { - checkKeywordsExist(sql("DESC EXTENDED ctas5"), - "name:key", "type:string", "name:value", "ctas5", - "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", - "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", - "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", - "MANAGED_TABLE" - ) - } - */ // use the Hive SerDe for parquet tables withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") { checkAnswer( sql("SELECT key, value FROM ctas5 ORDER BY key, value"), - sql("SELECT key, value FROM src ORDER BY key, value").collect().toSeq) + sql("SELECT key, value FROM src ORDER BY key, value")) } } From 36b501ebb18dca3195e44be92accd3fada479152 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Wed, 29 Mar 2017 14:55:10 -0700 Subject: [PATCH 11/15] fix. --- .../org/apache/spark/sql/catalyst/catalog/interface.scala | 3 ++- .../apache/spark/sql/hive/execution/HiveComparisonTest.scala | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 47cbca089110..4575dc49acf9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -20,9 +20,10 @@ package org.apache.spark.sql.catalyst.catalog import java.net.URI import java.util.Date -import com.google.common.base.Objects import scala.collection.mutable +import com.google.common.base.Objects + import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{CatalystConf, FunctionIdentifier, InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala index 313757706248..e45cf977bfaa 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala @@ -359,7 +359,7 @@ abstract class HiveComparisonTest stringToFile(new File(failedDirectory, testCaseName), errorMessage + consoleTestCase) fail(errorMessage) } - }.toSeq + } (queryList, hiveResults, catalystResults).zipped.foreach { case (query, hive, (hiveQuery, catalyst)) => @@ -370,6 +370,7 @@ abstract class HiveComparisonTest if ((!hiveQuery.logical.isInstanceOf[ExplainCommand]) && (!hiveQuery.logical.isInstanceOf[ShowFunctionsCommand]) && (!hiveQuery.logical.isInstanceOf[DescribeFunctionCommand]) && + (!hiveQuery.logical.isInstanceOf[DescribeTableCommand]) && preparedHive != catalyst) { val hivePrintOut = s"== HIVE - ${preparedHive.size} row(s) ==" +: preparedHive From e1160181cb6483f06e31b002c983f35052260c51 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Fri, 31 Mar 2017 20:31:02 -0700 Subject: [PATCH 12/15] address comments. --- .../org/apache/spark/sql/catalyst/catalog/interface.scala | 2 ++ .../org/apache/spark/sql/execution/command/tables.scala | 6 +++--- .../apache/spark/sql/execution/SparkSqlParserSuite.scala | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 4575dc49acf9..90dbc7e25634 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -115,6 +115,7 @@ case class CatalogTablePartition( }.mkString("CatalogPartition(\n\t", "\n\t", ")") } + /** Readable string representation for the CatalogTablePartition. */ def simpleString: String = { toLinkedHashMap.map { case ((key, value)) => if (value.isEmpty) key else s"$key: $value" @@ -326,6 +327,7 @@ case class CatalogTable( }.mkString("CatalogTable(\n", "\n", ")") } + /** Readable string representation for the CatalogTable. */ def simpleString: String = { toLinkedHashMap.map { case ((key, value)) => if (value.isEmpty) key else s"$key: $value" diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index b94a7bb151c4..1958d6f75fa6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -500,7 +500,7 @@ case class TruncateTableCommand( case class DescribeTableCommand( table: TableIdentifier, partitionSpec: TablePartitionSpec, - isFormatted: Boolean) + isExtended: Boolean) extends RunnableCommand { override val output: Seq[Attribute] = Seq( @@ -537,7 +537,7 @@ case class DescribeTableCommand( if (partitionSpec.nonEmpty) { describeDetailedPartitionInfo(sparkSession, catalog, metadata, result) - } else if (isFormatted) { + } else if (isExtended) { describeFormattedTableInfo(metadata, result) } } @@ -570,7 +570,7 @@ case class DescribeTableCommand( } DDLUtils.verifyPartitionProviderIsHive(spark, metadata, "DESC PARTITION") val partition = catalog.getPartition(table, partitionSpec) - if (isFormatted) describeFormattedDetailedPartitionInfo(table, metadata, partition, result) + if (isExtended) describeFormattedDetailedPartitionInfo(table, metadata, partition, result) } private def describeFormattedDetailedPartitionInfo( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala index 6aaeecadead6..908b955abbf0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala @@ -224,13 +224,13 @@ class SparkSqlParserSuite extends PlanTest { test("SPARK-17328 Fix NPE with EXPLAIN DESCRIBE TABLE") { assertEqual("describe table t", DescribeTableCommand( - TableIdentifier("t"), Map.empty, isFormatted = false)) + TableIdentifier("t"), Map.empty, isExtended = false)) assertEqual("describe table extended t", DescribeTableCommand( - TableIdentifier("t"), Map.empty, isFormatted = true)) + TableIdentifier("t"), Map.empty, isExtended = true)) assertEqual("describe table formatted t", DescribeTableCommand( - TableIdentifier("t"), Map.empty, isFormatted = true)) + TableIdentifier("t"), Map.empty, isExtended = true)) intercept("explain describe tables x", "Unsupported SQL statement") } From a6db8a32b6ad498dde89d0e6358034ece21a5f8f Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Sat, 1 Apr 2017 10:32:20 -0700 Subject: [PATCH 13/15] address comments. --- .../sql/catalyst/catalog/interface.scala | 3 +- .../spark/sql/execution/command/tables.scala | 13 ++- .../sql-tests/results/change-column.sql.out | 9 ++ .../sql-tests/results/describe.sql.out | 86 +++++++++---------- .../sql-tests/results/show-tables.sql.out | 26 +++--- .../apache/spark/sql/SQLQueryTestSuite.scala | 13 +-- 6 files changed, 85 insertions(+), 65 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 90dbc7e25634..3f25f9e7258f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -295,7 +295,8 @@ case class CatalogTable( val tableProperties = properties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") val partitionColumns = partitionColumnNames.map(quoteIdentifier).mkString("[", ", ", "]") - map.put("Table", identifier.quotedString) + identifier.database.foreach(map.put("Database", _)) + map.put("Table", identifier.table) if (owner.nonEmpty) map.put("Owner", owner) map.put("Created", new Date(createTime).toString) map.put("Last Access", new Date(lastAccessTime).toString) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 1958d6f75fa6..7b54182c44e6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -536,6 +536,8 @@ case class DescribeTableCommand( describePartitionInfo(metadata, result) if (partitionSpec.nonEmpty) { + // Outputs the partition-specific info for the DDL command: + // "DESCRIBE [EXTENDED|FORMATTED] table_name PARTITION (partitionVal*)" describeDetailedPartitionInfo(sparkSession, catalog, metadata, result) } else if (isExtended) { describeFormattedTableInfo(metadata, result) @@ -548,15 +550,21 @@ case class DescribeTableCommand( private def describePartitionInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { if (table.partitionColumnNames.nonEmpty) { append(buffer, "# Partition Information", "", "") - append(buffer, s"# ${output.head.name}", output(1).name, output(2).name) describeSchema(table.partitionSchema, buffer) } } private def describeFormattedTableInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = { + // The following information has been already shown in the previous outputs + val excludedTableInfo = Seq( + "Partition Columns", + "Schema" + ) append(buffer, "", "", "") append(buffer, "# Detailed Table Information", "", "") - table.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, "")) + table.toLinkedHashMap.filterKeys(!excludedTableInfo.contains(_)).foreach { + s => append(buffer, s._1, s._2, "") + } } private def describeDetailedPartitionInfo( @@ -594,6 +602,7 @@ case class DescribeTableCommand( } private def describeSchema(schema: StructType, buffer: ArrayBuffer[Row]): Unit = { + append(buffer, s"# ${output.head.name}", output(1).name, output(2).name) schema.foreach { column => append(buffer, column.name, column.dataType.simpleString, column.getComment().orNull) } diff --git a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out index ba8bc936f0c7..678a3f0f0a3c 100644 --- a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out @@ -15,6 +15,7 @@ DESC test_change -- !query 1 schema struct -- !query 1 output +# col_name data_type comment a int b string c int @@ -34,6 +35,7 @@ DESC test_change -- !query 3 schema struct -- !query 3 output +# col_name data_type comment a int b string c int @@ -53,6 +55,7 @@ DESC test_change -- !query 5 schema struct -- !query 5 output +# col_name data_type comment a int b string c int @@ -91,6 +94,7 @@ DESC test_change -- !query 8 schema struct -- !query 8 output +# col_name data_type comment a int b string c int @@ -125,6 +129,7 @@ DESC test_change -- !query 12 schema struct -- !query 12 output +# col_name data_type comment a int this is column a b string #*02?` c int @@ -143,6 +148,7 @@ DESC test_change -- !query 14 schema struct -- !query 14 output +# col_name data_type comment a int this is column a b string #*02?` c int @@ -162,6 +168,7 @@ DESC test_change -- !query 16 schema struct -- !query 16 output +# col_name data_type comment a int this is column a b string #*02?` c int @@ -186,6 +193,7 @@ DESC test_change -- !query 18 schema struct -- !query 18 output +# col_name data_type comment a int this is column a b string #*02?` c int @@ -229,6 +237,7 @@ DESC test_change -- !query 23 schema struct -- !query 23 output +# col_name data_type comment a int this is column A b string #*02?` c int diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index c7f4738f1ca7..4f91eb9fd7a7 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -41,6 +41,7 @@ DESCRIBE t -- !query 4 schema struct -- !query 4 output +# col_name data_type comment a string b int c string @@ -56,6 +57,7 @@ DESC t -- !query 5 schema struct -- !query 5 output +# col_name data_type comment a string b int c string @@ -71,6 +73,7 @@ DESC TABLE t -- !query 6 schema struct -- !query 6 output +# col_name data_type comment a string b int c string @@ -86,6 +89,7 @@ DESC FORMATTED t -- !query 7 schema struct -- !query 7 output +# col_name data_type comment a string b int c string @@ -96,23 +100,18 @@ c string d string # Detailed Table Information -Table `default`.`t` -Created[removed by SQLQueryTestSuite] -Last Access[removed by SQLQueryTestSuite] +Database default +Table t +Created [not included in comparison] +Last Access [not included in comparison] Type MANAGED Provider parquet Num Buckets 2 Bucket Columns [`a`] Sort Columns [`b`] Comment table_comment -Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t -Partition Provider Catalog -Partition Columns [`c`, `d`] -Schema root - |-- a: string (nullable = true) - |-- b: integer (nullable = true) - |-- c: string (nullable = true) - |-- d: string (nullable = true) +Location [not included in comparison]sql/core/spark-warehouse/t +Partition Provider Catalog -- !query 8 @@ -120,6 +119,7 @@ DESC EXTENDED t -- !query 8 schema struct -- !query 8 output +# col_name data_type comment a string b int c string @@ -130,23 +130,18 @@ c string d string # Detailed Table Information -Table `default`.`t` -Created[removed by SQLQueryTestSuite] -Last Access[removed by SQLQueryTestSuite] +Database default +Table t +Created [not included in comparison] +Last Access [not included in comparison] Type MANAGED Provider parquet Num Buckets 2 Bucket Columns [`a`] Sort Columns [`b`] Comment table_comment -Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t -Partition Provider Catalog -Partition Columns [`c`, `d`] -Schema root - |-- a: string (nullable = true) - |-- b: integer (nullable = true) - |-- c: string (nullable = true) - |-- d: string (nullable = true) +Location [not included in comparison]sql/core/spark-warehouse/t +Partition Provider Catalog -- !query 9 @@ -154,6 +149,7 @@ DESC t PARTITION (c='Us', d=1) -- !query 9 schema struct -- !query 9 output +# col_name data_type comment a string b int c string @@ -169,6 +165,7 @@ DESC EXTENDED t PARTITION (c='Us', d=1) -- !query 10 schema struct -- !query 10 output +# col_name data_type comment a string b int c string @@ -182,13 +179,13 @@ d string Database default Table t Partition Values [c=Us, d=1] -Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t/c=Us/d=1 +Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1 # Table Storage Information Num Buckets 2 Bucket Columns [`a`] Sort Columns [`b`] -Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t +Location [not included in comparison]sql/core/spark-warehouse/t -- !query 11 @@ -196,6 +193,7 @@ DESC FORMATTED t PARTITION (c='Us', d=1) -- !query 11 schema struct -- !query 11 output +# col_name data_type comment a string b int c string @@ -209,13 +207,13 @@ d string Database default Table t Partition Values [c=Us, d=1] -Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t/c=Us/d=1 +Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1 # Table Storage Information Num Buckets 2 Bucket Columns [`a`] Sort Columns [`b`] -Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/t +Location [not included in comparison]sql/core/spark-warehouse/t -- !query 12 @@ -257,6 +255,7 @@ DESC temp_v -- !query 15 schema struct -- !query 15 output +# col_name data_type comment a string b int c string @@ -268,6 +267,7 @@ DESC TABLE temp_v -- !query 16 schema struct -- !query 16 output +# col_name data_type comment a string b int c string @@ -279,6 +279,7 @@ DESC FORMATTED temp_v -- !query 17 schema struct -- !query 17 output +# col_name data_type comment a string b int c string @@ -290,6 +291,7 @@ DESC EXTENDED temp_v -- !query 18 schema struct -- !query 18 output +# col_name data_type comment a string b int c string @@ -310,6 +312,7 @@ DESC v -- !query 20 schema struct -- !query 20 output +# col_name data_type comment a string b int c string @@ -321,6 +324,7 @@ DESC TABLE v -- !query 21 schema struct -- !query 21 output +# col_name data_type comment a string b int c string @@ -332,25 +336,22 @@ DESC FORMATTED v -- !query 22 schema struct -- !query 22 output +# col_name data_type comment a string b int c string d string # Detailed Table Information -Table `default`.`v` -Created[removed by SQLQueryTestSuite] -Last Access[removed by SQLQueryTestSuite] +Database default +Table v +Created [not included in comparison] +Last Access [not included in comparison] Type VIEW View Text SELECT * FROM t View Default Database default View Query Output Columns [a, b, c, d] -Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] -Schema root - |-- a: string (nullable = true) - |-- b: integer (nullable = true) - |-- c: string (nullable = true) - |-- d: string (nullable = true) +Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] -- !query 23 @@ -358,25 +359,22 @@ DESC EXTENDED v -- !query 23 schema struct -- !query 23 output +# col_name data_type comment a string b int c string d string # Detailed Table Information -Table `default`.`v` -Created[removed by SQLQueryTestSuite] -Last Access[removed by SQLQueryTestSuite] +Database default +Table v +Created [not included in comparison] +Last Access [not included in comparison] Type VIEW View Text SELECT * FROM t View Default Database default View Query Output Columns [a, b, c, d] -Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] -Schema root - |-- a: string (nullable = true) - |-- b: integer (nullable = true) - |-- c: string (nullable = true) - |-- d: string (nullable = true) +Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] -- !query 24 diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out index 10923a25c971..8f2a54f7c24e 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out @@ -118,20 +118,21 @@ SHOW TABLE EXTENDED LIKE 'show_t*' -- !query 12 schema struct -- !query 12 output -show_t3 true Table: `show_t3` -Created[removed by SQLQueryTestSuite] -Last Access[removed by SQLQueryTestSuite] +show_t3 true Table: show_t3 +Created [not included in comparison] +Last Access [not included in comparison] Type: VIEW Schema: root |-- e: integer (nullable = true) -showdb show_t1 false Table: `showdb`.`show_t1` -Created[removed by SQLQueryTestSuite] -Last Access[removed by SQLQueryTestSuite] +showdb show_t1 false Database: showdb +Table: show_t1 +Created [not included in comparison] +Last Access [not included in comparison] Type: MANAGED Provider: parquet -Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t1 +Location [not included in comparison]sql/core/spark-warehouse/showdb.db/show_t1 Partition Provider: Catalog Partition Columns: [`c`, `d`] Schema: root @@ -141,12 +142,13 @@ Schema: root |-- d: string (nullable = true) -showdb show_t2 false Table: `showdb`.`show_t2` -Created[removed by SQLQueryTestSuite] -Last Access[removed by SQLQueryTestSuite] +showdb show_t2 false Database: showdb +Table: show_t2 +Created [not included in comparison] +Last Access [not included in comparison] Type: MANAGED Provider: parquet -Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t2 +Location [not included in comparison]sql/core/spark-warehouse/showdb.db/show_t2 Schema: root |-- b: string (nullable = true) |-- d: integer (nullable = true) @@ -172,7 +174,7 @@ SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us', d=1) struct -- !query 14 output showdb show_t1 false Partition Values: [c=Us, d=1] -Location[removed by SQLQueryTestSuite]sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1 +Location [not included in comparison]sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1 -- !query 15 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index d4474efe6608..4b69baffab62 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -213,24 +213,25 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { /** Executes a query and returns the result as (schema of the output, normalized output). */ private def getNormalizedResult(session: SparkSession, sql: String): (StructType, Seq[String]) = { // Returns true if the plan is supposed to be sorted. - def isSorted(plan: LogicalPlan): Boolean = plan match { + def needSort(plan: LogicalPlan): Boolean = plan match { case _: Join | _: Aggregate | _: Generate | _: Sample | _: Distinct => false case _: DescribeTableCommand => true case PhysicalOperation(_, _, Sort(_, true, _)) => true - case _ => plan.children.iterator.exists(isSorted) + case _ => plan.children.iterator.exists(needSort) } try { val df = session.sql(sql) val schema = df.schema + val notIncludedMsg = "[not included in comparison]" // Get answer, but also get rid of the #1234 expression ids that show up in explain plans val answer = df.queryExecution.hiveResultString().map(_.replaceAll("#\\d+", "#x") - .replaceAll("Location.*/sql/core/", "Location[removed by SQLQueryTestSuite]sql/core/") - .replaceAll("Created.*", "Created[removed by SQLQueryTestSuite]") - .replaceAll("Last Access.*", "Last Access[removed by SQLQueryTestSuite]")) + .replaceAll("Location.*/sql/core/", s"Location ${notIncludedMsg}sql/core/") + .replaceAll("Created.*", s"Created $notIncludedMsg") + .replaceAll("Last Access.*", s"Last Access $notIncludedMsg")) // If the output is not pre-sorted, sort it. - if (isSorted(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted) + if (needSort(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted) } catch { case a: AnalysisException => From 43668be3b290b61129162fee27d13a73cece794a Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Sat, 1 Apr 2017 18:09:41 -0700 Subject: [PATCH 14/15] fix the test cases --- .../resources/sql-tests/inputs/describe.sql | 13 +- .../sql-tests/results/describe.sql.out | 195 +++++++++++------- .../sql/execution/command/DDLSuite.scala | 12 -- .../spark/sql/sources/DDLTestSuite.scala | 123 ----------- .../spark/sql/sources/DataSourceTest.scala | 56 +++++ .../sql/hive/MetastoreDataSourcesSuite.scala | 8 +- .../sql/hive/execution/HiveDDLSuite.scala | 2 +- .../HiveOperatorQueryableSuite.scala | 53 ----- .../sql/hive/execution/HiveQuerySuite.scala | 56 ----- 9 files changed, 193 insertions(+), 325 deletions(-) delete mode 100644 sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala delete mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql index 3830616da82c..6de4cf0d5afa 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql @@ -4,13 +4,20 @@ CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet CREATE TEMPORARY VIEW temp_v AS SELECT * FROM t; +CREATE TEMPORARY VIEW temp_Data_Source_View + USING org.apache.spark.sql.sources.DDLScanSource + OPTIONS ( + From '1', + To '10', + Table 'test1'); + CREATE VIEW v AS SELECT * FROM t; ALTER TABLE t ADD PARTITION (c='Us', d=1); DESCRIBE t; -DESC t; +DESC default.t; DESC TABLE t; @@ -43,6 +50,8 @@ DESC FORMATTED temp_v; DESC EXTENDED temp_v; +DESC temp_Data_Source_View; + -- AnalysisException DESC PARTITION is not allowed on a temporary view DESC temp_v PARTITION (c='Us', d=1); @@ -64,4 +73,6 @@ DROP TABLE t; DROP VIEW temp_v; +DROP VIEW temp_Data_Source_View; + DROP VIEW v; diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 4f91eb9fd7a7..d1916fb57227 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 28 +-- Number of queries: 31 -- !query 0 @@ -21,7 +21,12 @@ struct<> -- !query 2 -CREATE VIEW v AS SELECT * FROM t +CREATE TEMPORARY VIEW temp_Data_Source_View + USING org.apache.spark.sql.sources.DDLScanSource + OPTIONS ( + From '1', + To '10', + Table 'test1') -- !query 2 schema struct<> -- !query 2 output @@ -29,7 +34,7 @@ struct<> -- !query 3 -ALTER TABLE t ADD PARTITION (c='Us', d=1) +CREATE VIEW v AS SELECT * FROM t -- !query 3 schema struct<> -- !query 3 output @@ -37,10 +42,18 @@ struct<> -- !query 4 -DESCRIBE t +ALTER TABLE t ADD PARTITION (c='Us', d=1) -- !query 4 schema -struct +struct<> -- !query 4 output + + + +-- !query 5 +DESCRIBE t +-- !query 5 schema +struct +-- !query 5 output # col_name data_type comment a string b int @@ -52,11 +65,11 @@ c string d string --- !query 5 -DESC t --- !query 5 schema +-- !query 6 +DESC default.t +-- !query 6 schema struct --- !query 5 output +-- !query 6 output # col_name data_type comment a string b int @@ -68,11 +81,11 @@ c string d string --- !query 6 +-- !query 7 DESC TABLE t --- !query 6 schema +-- !query 7 schema struct --- !query 6 output +-- !query 7 output # col_name data_type comment a string b int @@ -84,11 +97,11 @@ c string d string --- !query 7 +-- !query 8 DESC FORMATTED t --- !query 7 schema +-- !query 8 schema struct --- !query 7 output +-- !query 8 output # col_name data_type comment a string b int @@ -114,11 +127,11 @@ Location [not included in comparison]sql/core/spark-warehouse/t Partition Provider Catalog --- !query 8 +-- !query 9 DESC EXTENDED t --- !query 8 schema +-- !query 9 schema struct --- !query 8 output +-- !query 9 output # col_name data_type comment a string b int @@ -144,11 +157,11 @@ Location [not included in comparison]sql/core/spark-warehouse/t Partition Provider Catalog --- !query 9 +-- !query 10 DESC t PARTITION (c='Us', d=1) --- !query 9 schema +-- !query 10 schema struct --- !query 9 output +-- !query 10 output # col_name data_type comment a string b int @@ -160,11 +173,11 @@ c string d string --- !query 10 +-- !query 11 DESC EXTENDED t PARTITION (c='Us', d=1) --- !query 10 schema +-- !query 11 schema struct --- !query 10 output +-- !query 11 output # col_name data_type comment a string b int @@ -188,11 +201,11 @@ Sort Columns [`b`] Location [not included in comparison]sql/core/spark-warehouse/t --- !query 11 +-- !query 12 DESC FORMATTED t PARTITION (c='Us', d=1) --- !query 11 schema +-- !query 12 schema struct --- !query 11 output +-- !query 12 output # col_name data_type comment a string b int @@ -216,31 +229,31 @@ Sort Columns [`b`] Location [not included in comparison]sql/core/spark-warehouse/t --- !query 12 +-- !query 13 DESC t PARTITION (c='Us', d=2) --- !query 12 schema +-- !query 13 schema struct<> --- !query 12 output +-- !query 13 output org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException Partition not found in table 't' database 'default': c -> Us d -> 2; --- !query 13 +-- !query 14 DESC t PARTITION (c='Us') --- !query 13 schema +-- !query 14 schema struct<> --- !query 13 output +-- !query 14 output org.apache.spark.sql.AnalysisException Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`'; --- !query 14 +-- !query 15 DESC t PARTITION (c='Us', d) --- !query 14 schema +-- !query 15 schema struct<> --- !query 14 output +-- !query 15 output org.apache.spark.sql.catalyst.parser.ParseException PARTITION specification is incomplete: `d`(line 1, pos 0) @@ -250,11 +263,11 @@ DESC t PARTITION (c='Us', d) ^^^ --- !query 15 +-- !query 16 DESC temp_v --- !query 15 schema +-- !query 16 schema struct --- !query 15 output +-- !query 16 output # col_name data_type comment a string b int @@ -262,11 +275,11 @@ c string d string --- !query 16 +-- !query 17 DESC TABLE temp_v --- !query 16 schema +-- !query 17 schema struct --- !query 16 output +-- !query 17 output # col_name data_type comment a string b int @@ -274,11 +287,11 @@ c string d string --- !query 17 +-- !query 18 DESC FORMATTED temp_v --- !query 17 schema +-- !query 18 schema struct --- !query 17 output +-- !query 18 output # col_name data_type comment a string b int @@ -286,11 +299,11 @@ c string d string --- !query 18 +-- !query 19 DESC EXTENDED temp_v --- !query 18 schema +-- !query 19 schema struct --- !query 18 output +-- !query 19 output # col_name data_type comment a string b int @@ -298,20 +311,44 @@ c string d string --- !query 19 +-- !query 20 +DESC temp_Data_Source_View +-- !query 20 schema +struct +-- !query 20 output +# col_name data_type comment +intType int test comment test1 +stringType string +dateType date +timestampType timestamp +doubleType double +bigintType bigint +tinyintType tinyint +decimalType decimal(10,0) +fixedDecimalType decimal(5,1) +binaryType binary +booleanType boolean +smallIntType smallint +floatType float +mapType map +arrayType array +structType struct + + +-- !query 21 DESC temp_v PARTITION (c='Us', d=1) --- !query 19 schema +-- !query 21 schema struct<> --- !query 19 output +-- !query 21 output org.apache.spark.sql.AnalysisException DESC PARTITION is not allowed on a temporary view: temp_v; --- !query 20 +-- !query 22 DESC v --- !query 20 schema +-- !query 22 schema struct --- !query 20 output +-- !query 22 output # col_name data_type comment a string b int @@ -319,11 +356,11 @@ c string d string --- !query 21 +-- !query 23 DESC TABLE v --- !query 21 schema +-- !query 23 schema struct --- !query 21 output +-- !query 23 output # col_name data_type comment a string b int @@ -331,11 +368,11 @@ c string d string --- !query 22 +-- !query 24 DESC FORMATTED v --- !query 22 schema +-- !query 24 schema struct --- !query 22 output +-- !query 24 output # col_name data_type comment a string b int @@ -354,11 +391,11 @@ View Query Output Columns [a, b, c, d] Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] --- !query 23 +-- !query 25 DESC EXTENDED v --- !query 23 schema +-- !query 25 schema struct --- !query 23 output +-- !query 25 output # col_name data_type comment a string b int @@ -377,34 +414,42 @@ View Query Output Columns [a, b, c, d] Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c] --- !query 24 +-- !query 26 DESC v PARTITION (c='Us', d=1) --- !query 24 schema +-- !query 26 schema struct<> --- !query 24 output +-- !query 26 output org.apache.spark.sql.AnalysisException DESC PARTITION is not allowed on a view: v; --- !query 25 +-- !query 27 DROP TABLE t --- !query 25 schema +-- !query 27 schema struct<> --- !query 25 output +-- !query 27 output --- !query 26 +-- !query 28 DROP VIEW temp_v --- !query 26 schema +-- !query 28 schema struct<> --- !query 26 output +-- !query 28 output --- !query 27 +-- !query 29 +DROP VIEW temp_Data_Source_View +-- !query 29 schema +struct<> +-- !query 29 output + + + +-- !query 30 DROP VIEW v --- !query 27 schema +-- !query 30 schema struct<> --- !query 27 output +-- !query 30 output diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 648b1798c66e..9ebf2dd839a7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -69,18 +69,6 @@ class InMemoryCatalogedDDLSuite extends DDLSuite with SharedSQLContext with Befo tracksPartitionsInCatalog = true) } - test("desc table for parquet data source table using in-memory catalog") { - val tabName = "tab1" - withTable(tabName) { - sql(s"CREATE TABLE $tabName(a int comment 'test') USING parquet ") - - checkAnswer( - sql(s"DESC $tabName").select("col_name", "data_type", "comment"), - Row("a", "int", "test") - ) - } - } - test("alter table: set location (datasource table)") { testSetLocation(isDatasourceTable = true) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala deleted file mode 100644 index 674463feca4d..000000000000 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala +++ /dev/null @@ -1,123 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.spark.sql.sources - -import org.apache.spark.rdd.RDD -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.test.SharedSQLContext -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.UTF8String - -class DDLScanSource extends RelationProvider { - override def createRelation( - sqlContext: SQLContext, - parameters: Map[String, String]): BaseRelation = { - SimpleDDLScan( - parameters("from").toInt, - parameters("TO").toInt, - parameters("Table"))(sqlContext.sparkSession) - } -} - -case class SimpleDDLScan( - from: Int, - to: Int, - table: String)(@transient val sparkSession: SparkSession) - extends BaseRelation with TableScan { - - override def sqlContext: SQLContext = sparkSession.sqlContext - - override def schema: StructType = - StructType(Seq( - StructField("intType", IntegerType, nullable = false).withComment(s"test comment $table"), - StructField("stringType", StringType, nullable = false), - StructField("dateType", DateType, nullable = false), - StructField("timestampType", TimestampType, nullable = false), - StructField("doubleType", DoubleType, nullable = false), - StructField("bigintType", LongType, nullable = false), - StructField("tinyintType", ByteType, nullable = false), - StructField("decimalType", DecimalType.USER_DEFAULT, nullable = false), - StructField("fixedDecimalType", DecimalType(5, 1), nullable = false), - StructField("binaryType", BinaryType, nullable = false), - StructField("booleanType", BooleanType, nullable = false), - StructField("smallIntType", ShortType, nullable = false), - StructField("floatType", FloatType, nullable = false), - StructField("mapType", MapType(StringType, StringType)), - StructField("arrayType", ArrayType(StringType)), - StructField("structType", - StructType(StructField("f1", StringType) :: StructField("f2", IntegerType) :: Nil - ) - ) - )) - - override def needConversion: Boolean = false - - override def buildScan(): RDD[Row] = { - // Rely on a type erasure hack to pass RDD[InternalRow] back as RDD[Row] - sparkSession.sparkContext.parallelize(from to to).map { e => - InternalRow(UTF8String.fromString(s"people$e"), e * 2) - }.asInstanceOf[RDD[Row]] - } -} - -class DDLTestSuite extends DataSourceTest with SharedSQLContext { - protected override lazy val sql = spark.sql _ - - override def beforeAll(): Unit = { - super.beforeAll() - sql( - """ - |CREATE OR REPLACE TEMPORARY VIEW ddlPeople - |USING org.apache.spark.sql.sources.DDLScanSource - |OPTIONS ( - | From '1', - | To '10', - | Table 'test1' - |) - """.stripMargin) - } - - sqlTest( - "describe ddlPeople", - Seq( - Row("intType", "int", "test comment test1"), - Row("stringType", "string", null), - Row("dateType", "date", null), - Row("timestampType", "timestamp", null), - Row("doubleType", "double", null), - Row("bigintType", "bigint", null), - Row("tinyintType", "tinyint", null), - Row("decimalType", "decimal(10,0)", null), - Row("fixedDecimalType", "decimal(5,1)", null), - Row("binaryType", "binary", null), - Row("booleanType", "boolean", null), - Row("smallIntType", "smallint", null), - Row("floatType", "float", null), - Row("mapType", "map", null), - Row("arrayType", "array", null), - Row("structType", "struct", null) - )) - - test("SPARK-7686 DescribeCommand should have correct physical plan output attributes") { - val attributes = sql("describe ddlPeople") - .queryExecution.executedPlan.output - assert(attributes.map(_.name) === Seq("col_name", "data_type", "comment")) - assert(attributes.map(_.dataType).toSet === Set(StringType)) - } -} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceTest.scala index cc77d3c4b91a..80868fff897f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceTest.scala @@ -17,7 +17,11 @@ package org.apache.spark.sql.sources +import org.apache.spark.rdd.RDD import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.types._ +import org.apache.spark.unsafe.types.UTF8String private[sql] abstract class DataSourceTest extends QueryTest { @@ -28,3 +32,55 @@ private[sql] abstract class DataSourceTest extends QueryTest { } } + +class DDLScanSource extends RelationProvider { + override def createRelation( + sqlContext: SQLContext, + parameters: Map[String, String]): BaseRelation = { + SimpleDDLScan( + parameters("from").toInt, + parameters("TO").toInt, + parameters("Table"))(sqlContext.sparkSession) + } +} + +case class SimpleDDLScan( + from: Int, + to: Int, + table: String)(@transient val sparkSession: SparkSession) + extends BaseRelation with TableScan { + + override def sqlContext: SQLContext = sparkSession.sqlContext + + override def schema: StructType = + StructType(Seq( + StructField("intType", IntegerType, nullable = false).withComment(s"test comment $table"), + StructField("stringType", StringType, nullable = false), + StructField("dateType", DateType, nullable = false), + StructField("timestampType", TimestampType, nullable = false), + StructField("doubleType", DoubleType, nullable = false), + StructField("bigintType", LongType, nullable = false), + StructField("tinyintType", ByteType, nullable = false), + StructField("decimalType", DecimalType.USER_DEFAULT, nullable = false), + StructField("fixedDecimalType", DecimalType(5, 1), nullable = false), + StructField("binaryType", BinaryType, nullable = false), + StructField("booleanType", BooleanType, nullable = false), + StructField("smallIntType", ShortType, nullable = false), + StructField("floatType", FloatType, nullable = false), + StructField("mapType", MapType(StringType, StringType)), + StructField("arrayType", ArrayType(StringType)), + StructField("structType", + StructType(StructField("f1", StringType) :: StructField("f2", IntegerType) :: Nil + ) + ) + )) + + override def needConversion: Boolean = false + + override def buildScan(): RDD[Row] = { + // Rely on a type erasure hack to pass RDD[InternalRow] back as RDD[Row] + sparkSession.sparkContext.parallelize(from to to).map { e => + InternalRow(UTF8String.fromString(s"people$e"), e * 2) + }.asInstanceOf[RDD[Row]] + } +} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index f02b7218d6ee..7a56e29037be 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -768,9 +768,6 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv sessionState.refreshTable(tableName) val actualSchema = table(tableName).schema assert(schema === actualSchema) - - // Checks the DESCRIBE output. - checkAnswer(sql("DESCRIBE spark6655"), Row("int", "int", null) :: Nil) } } @@ -1382,7 +1379,10 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv checkAnswer(spark.table("old"), Row(1, "a")) - checkAnswer(sql("DESC old"), Row("i", "int", null) :: Row("j", "string", null) :: Nil) + val expectedSchema = StructType(Seq( + StructField("i", IntegerType, nullable = true), + StructField("j", StringType, nullable = true))) + assert(table("old").schema === expectedSchema) } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 1f9a35612c2d..20f11602f1e3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -732,7 +732,7 @@ class HiveDDLSuite checkAnswer( sql(s"DESC $tabName").select("col_name", "data_type", "comment"), - Row("a", "int", "test") + Row("# col_name", "data_type", "comment") :: Row("a", "int", "test") :: Nil ) } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala deleted file mode 100644 index 0e89e990e564..000000000000 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveOperatorQueryableSuite.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.{QueryTest, Row} -import org.apache.spark.sql.hive.test.TestHiveSingleton - -/** - * A set of tests that validates commands can also be queried by like a table - */ -class HiveOperatorQueryableSuite extends QueryTest with TestHiveSingleton { - import spark._ - - test("SPARK-5324 query result of describe command") { - hiveContext.loadTestTable("src") - - // Creates a temporary view with the output of a describe command - sql("desc src").createOrReplaceTempView("mydesc") - checkAnswer( - sql("desc mydesc"), - Seq( - Row("col_name", "string", "name of the column"), - Row("data_type", "string", "data type of the column"), - Row("comment", "string", "comment of the column"))) - - checkAnswer( - sql("select * from mydesc"), - Seq( - Row("key", "int", null), - Row("value", "string", null))) - - checkAnswer( - sql("select col_name, data_type, comment from mydesc"), - Seq( - Row("key", "int", null), - Row("value", "string", null))) - } -} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index dd278f683a3c..65a902fc5438 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -789,62 +789,6 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd assert(Try(q0.count()).isSuccess) } - test("DESCRIBE commands") { - sql(s"CREATE TABLE test_describe_commands1 (key INT, value STRING) PARTITIONED BY (dt STRING)") - - sql( - """FROM src INSERT OVERWRITE TABLE test_describe_commands1 PARTITION (dt='2008-06-08') - |SELECT key, value - """.stripMargin) - - // Describe a table - assertResult( - Array( - Row("key", "int", null), - Row("value", "string", null), - Row("dt", "string", null), - Row("# Partition Information", "", ""), - Row("# col_name", "data_type", "comment"), - Row("dt", "string", null)) - ) { - sql("DESCRIBE test_describe_commands1") - .select('col_name, 'data_type, 'comment) - .collect() - } - - // Describe a table with a fully qualified table name - assertResult( - Array( - Row("key", "int", null), - Row("value", "string", null), - Row("dt", "string", null), - Row("# Partition Information", "", ""), - Row("# col_name", "data_type", "comment"), - Row("dt", "string", null)) - ) { - sql("DESCRIBE default.test_describe_commands1") - .select('col_name, 'data_type, 'comment) - .collect() - } - - // Describe a temporary view. - val testData = - TestHive.sparkContext.parallelize( - TestData(1, "str1") :: - TestData(1, "str2") :: Nil) - testData.toDF().createOrReplaceTempView("test_describe_commands2") - - assertResult( - Array( - Row("a", "int", null), - Row("b", "string", null)) - ) { - sql("DESCRIBE test_describe_commands2") - .select('col_name, 'data_type, 'comment) - .collect() - } - } - test("SPARK-2263: Insert Map values") { sql("CREATE TABLE m(value MAP)") sql("INSERT OVERWRITE TABLE m SELECT MAP(key, value) FROM src LIMIT 10") From 862a4d7a61e48ff7b0e1d52ea0416bc57a4d6a33 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Mon, 3 Apr 2017 18:06:45 -0700 Subject: [PATCH 15/15] address comments. --- .../scala/org/apache/spark/sql/execution/command/tables.scala | 2 +- .../src/test/resources/sql-tests/results/describe.sql.out | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 7b54182c44e6..ebf03e1bf886 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -592,7 +592,7 @@ case class DescribeTableCommand( append(buffer, "Table", tableIdentifier.table, "") partition.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, "")) append(buffer, "", "", "") - append(buffer, "# Table Storage Information", "", "") + append(buffer, "# Storage Information", "", "") table.bucketSpec match { case Some(spec) => spec.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, "")) diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index d1916fb57227..de10b29f3c65 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -194,7 +194,7 @@ Table t Partition Values [c=Us, d=1] Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1 -# Table Storage Information +# Storage Information Num Buckets 2 Bucket Columns [`a`] Sort Columns [`b`] @@ -222,7 +222,7 @@ Table t Partition Values [c=Us, d=1] Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1 -# Table Storage Information +# Storage Information Num Buckets 2 Bucket Columns [`a`] Sort Columns [`b`]